1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 * Eddie Dong <eddie.dong@intel.com> 30 * 31 * Contributors: 32 * Niu Bing <bing.niu@intel.com> 33 * Zhi Wang <zhi.a.wang@intel.com> 34 */ 35 36 #include <linux/init.h> 37 #include <linux/mm.h> 38 #include <linux/kthread.h> 39 #include <linux/sched/mm.h> 40 #include <linux/types.h> 41 #include <linux/list.h> 42 #include <linux/rbtree.h> 43 #include <linux/spinlock.h> 44 #include <linux/eventfd.h> 45 #include <linux/mdev.h> 46 #include <linux/debugfs.h> 47 48 #include <linux/nospec.h> 49 50 #include <drm/drm_edid.h> 51 52 #include "i915_drv.h" 53 #include "intel_gvt.h" 54 #include "gvt.h" 55 56 MODULE_IMPORT_NS(DMA_BUF); 57 MODULE_IMPORT_NS(I915_GVT); 58 59 /* helper macros copied from vfio-pci */ 60 #define VFIO_PCI_OFFSET_SHIFT 40 61 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 62 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 63 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 64 65 #define EDID_BLOB_OFFSET (PAGE_SIZE/2) 66 67 #define OPREGION_SIGNATURE "IntelGraphicsMem" 68 69 struct vfio_region; 70 struct intel_vgpu_regops { 71 size_t (*rw)(struct intel_vgpu *vgpu, char *buf, 72 size_t count, loff_t *ppos, bool iswrite); 73 void (*release)(struct intel_vgpu *vgpu, 74 struct vfio_region *region); 75 }; 76 77 struct vfio_region { 78 u32 type; 79 u32 subtype; 80 size_t size; 81 u32 flags; 82 const struct intel_vgpu_regops *ops; 83 void *data; 84 }; 85 86 struct vfio_edid_region { 87 struct vfio_region_gfx_edid vfio_edid_regs; 88 void *edid_blob; 89 }; 90 91 struct kvmgt_pgfn { 92 gfn_t gfn; 93 struct hlist_node hnode; 94 }; 95 96 struct gvt_dma { 97 struct intel_vgpu *vgpu; 98 struct rb_node gfn_node; 99 struct rb_node dma_addr_node; 100 gfn_t gfn; 101 dma_addr_t dma_addr; 102 unsigned long size; 103 struct kref ref; 104 }; 105 106 #define vfio_dev_to_vgpu(vfio_dev) \ 107 container_of((vfio_dev), struct intel_vgpu, vfio_device) 108 109 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 110 const u8 *val, int len, 111 struct kvm_page_track_notifier_node *node); 112 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 113 struct kvm_memory_slot *slot, 114 struct kvm_page_track_notifier_node *node); 115 116 static ssize_t available_instances_show(struct mdev_type *mtype, 117 struct mdev_type_attribute *attr, 118 char *buf) 119 { 120 struct intel_vgpu_type *type; 121 unsigned int num = 0; 122 struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; 123 124 type = &gvt->types[mtype_get_type_group_id(mtype)]; 125 if (!type) 126 num = 0; 127 else 128 num = type->avail_instance; 129 130 return sprintf(buf, "%u\n", num); 131 } 132 133 static ssize_t device_api_show(struct mdev_type *mtype, 134 struct mdev_type_attribute *attr, char *buf) 135 { 136 return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); 137 } 138 139 static ssize_t description_show(struct mdev_type *mtype, 140 struct mdev_type_attribute *attr, char *buf) 141 { 142 struct intel_vgpu_type *type; 143 struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; 144 145 type = &gvt->types[mtype_get_type_group_id(mtype)]; 146 if (!type) 147 return 0; 148 149 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" 150 "fence: %d\nresolution: %s\n" 151 "weight: %d\n", 152 BYTES_TO_MB(type->conf->low_mm), 153 BYTES_TO_MB(type->conf->high_mm), 154 type->conf->fence, vgpu_edid_str(type->conf->edid), 155 type->conf->weight); 156 } 157 158 static ssize_t name_show(struct mdev_type *mtype, 159 struct mdev_type_attribute *attr, char *buf) 160 { 161 struct intel_vgpu_type *type; 162 struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; 163 164 type = &gvt->types[mtype_get_type_group_id(mtype)]; 165 if (!type) 166 return 0; 167 168 return sprintf(buf, "%s\n", type->name); 169 } 170 171 static MDEV_TYPE_ATTR_RO(available_instances); 172 static MDEV_TYPE_ATTR_RO(device_api); 173 static MDEV_TYPE_ATTR_RO(description); 174 static MDEV_TYPE_ATTR_RO(name); 175 176 static struct attribute *gvt_type_attrs[] = { 177 &mdev_type_attr_available_instances.attr, 178 &mdev_type_attr_device_api.attr, 179 &mdev_type_attr_description.attr, 180 &mdev_type_attr_name.attr, 181 NULL, 182 }; 183 184 static struct attribute_group *gvt_vgpu_type_groups[] = { 185 [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, 186 }; 187 188 static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) 189 { 190 int i, j; 191 struct intel_vgpu_type *type; 192 struct attribute_group *group; 193 194 for (i = 0; i < gvt->num_types; i++) { 195 type = &gvt->types[i]; 196 197 group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); 198 if (!group) 199 goto unwind; 200 201 group->name = type->name; 202 group->attrs = gvt_type_attrs; 203 gvt_vgpu_type_groups[i] = group; 204 } 205 206 return 0; 207 208 unwind: 209 for (j = 0; j < i; j++) { 210 group = gvt_vgpu_type_groups[j]; 211 kfree(group); 212 } 213 214 return -ENOMEM; 215 } 216 217 static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) 218 { 219 int i; 220 struct attribute_group *group; 221 222 for (i = 0; i < gvt->num_types; i++) { 223 group = gvt_vgpu_type_groups[i]; 224 gvt_vgpu_type_groups[i] = NULL; 225 kfree(group); 226 } 227 } 228 229 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 230 unsigned long size) 231 { 232 vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT, 233 DIV_ROUND_UP(size, PAGE_SIZE)); 234 } 235 236 /* Pin a normal or compound guest page for dma. */ 237 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 238 unsigned long size, struct page **page) 239 { 240 int total_pages = DIV_ROUND_UP(size, PAGE_SIZE); 241 struct page *base_page = NULL; 242 int npage; 243 int ret; 244 245 /* 246 * We pin the pages one-by-one to avoid allocating a big arrary 247 * on stack to hold pfns. 248 */ 249 for (npage = 0; npage < total_pages; npage++) { 250 dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT; 251 struct page *cur_page; 252 253 ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1, 254 IOMMU_READ | IOMMU_WRITE, &cur_page); 255 if (ret != 1) { 256 gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n", 257 &cur_iova, ret); 258 goto err; 259 } 260 261 if (npage == 0) 262 base_page = cur_page; 263 else if (base_page + npage != cur_page) { 264 gvt_vgpu_err("The pages are not continuous\n"); 265 ret = -EINVAL; 266 npage++; 267 goto err; 268 } 269 } 270 271 *page = base_page; 272 return 0; 273 err: 274 gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE); 275 return ret; 276 } 277 278 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn, 279 dma_addr_t *dma_addr, unsigned long size) 280 { 281 struct device *dev = vgpu->gvt->gt->i915->drm.dev; 282 struct page *page = NULL; 283 int ret; 284 285 ret = gvt_pin_guest_page(vgpu, gfn, size, &page); 286 if (ret) 287 return ret; 288 289 /* Setup DMA mapping. */ 290 *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL); 291 if (dma_mapping_error(dev, *dma_addr)) { 292 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n", 293 page_to_pfn(page), ret); 294 gvt_unpin_guest_page(vgpu, gfn, size); 295 return -ENOMEM; 296 } 297 298 return 0; 299 } 300 301 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn, 302 dma_addr_t dma_addr, unsigned long size) 303 { 304 struct device *dev = vgpu->gvt->gt->i915->drm.dev; 305 306 dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL); 307 gvt_unpin_guest_page(vgpu, gfn, size); 308 } 309 310 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu, 311 dma_addr_t dma_addr) 312 { 313 struct rb_node *node = vgpu->dma_addr_cache.rb_node; 314 struct gvt_dma *itr; 315 316 while (node) { 317 itr = rb_entry(node, struct gvt_dma, dma_addr_node); 318 319 if (dma_addr < itr->dma_addr) 320 node = node->rb_left; 321 else if (dma_addr > itr->dma_addr) 322 node = node->rb_right; 323 else 324 return itr; 325 } 326 return NULL; 327 } 328 329 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn) 330 { 331 struct rb_node *node = vgpu->gfn_cache.rb_node; 332 struct gvt_dma *itr; 333 334 while (node) { 335 itr = rb_entry(node, struct gvt_dma, gfn_node); 336 337 if (gfn < itr->gfn) 338 node = node->rb_left; 339 else if (gfn > itr->gfn) 340 node = node->rb_right; 341 else 342 return itr; 343 } 344 return NULL; 345 } 346 347 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 348 dma_addr_t dma_addr, unsigned long size) 349 { 350 struct gvt_dma *new, *itr; 351 struct rb_node **link, *parent = NULL; 352 353 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 354 if (!new) 355 return -ENOMEM; 356 357 new->vgpu = vgpu; 358 new->gfn = gfn; 359 new->dma_addr = dma_addr; 360 new->size = size; 361 kref_init(&new->ref); 362 363 /* gfn_cache maps gfn to struct gvt_dma. */ 364 link = &vgpu->gfn_cache.rb_node; 365 while (*link) { 366 parent = *link; 367 itr = rb_entry(parent, struct gvt_dma, gfn_node); 368 369 if (gfn < itr->gfn) 370 link = &parent->rb_left; 371 else 372 link = &parent->rb_right; 373 } 374 rb_link_node(&new->gfn_node, parent, link); 375 rb_insert_color(&new->gfn_node, &vgpu->gfn_cache); 376 377 /* dma_addr_cache maps dma addr to struct gvt_dma. */ 378 parent = NULL; 379 link = &vgpu->dma_addr_cache.rb_node; 380 while (*link) { 381 parent = *link; 382 itr = rb_entry(parent, struct gvt_dma, dma_addr_node); 383 384 if (dma_addr < itr->dma_addr) 385 link = &parent->rb_left; 386 else 387 link = &parent->rb_right; 388 } 389 rb_link_node(&new->dma_addr_node, parent, link); 390 rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache); 391 392 vgpu->nr_cache_entries++; 393 return 0; 394 } 395 396 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 397 struct gvt_dma *entry) 398 { 399 rb_erase(&entry->gfn_node, &vgpu->gfn_cache); 400 rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache); 401 kfree(entry); 402 vgpu->nr_cache_entries--; 403 } 404 405 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 406 { 407 struct gvt_dma *dma; 408 struct rb_node *node = NULL; 409 410 for (;;) { 411 mutex_lock(&vgpu->cache_lock); 412 node = rb_first(&vgpu->gfn_cache); 413 if (!node) { 414 mutex_unlock(&vgpu->cache_lock); 415 break; 416 } 417 dma = rb_entry(node, struct gvt_dma, gfn_node); 418 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size); 419 __gvt_cache_remove_entry(vgpu, dma); 420 mutex_unlock(&vgpu->cache_lock); 421 } 422 } 423 424 static void gvt_cache_init(struct intel_vgpu *vgpu) 425 { 426 vgpu->gfn_cache = RB_ROOT; 427 vgpu->dma_addr_cache = RB_ROOT; 428 vgpu->nr_cache_entries = 0; 429 mutex_init(&vgpu->cache_lock); 430 } 431 432 static void kvmgt_protect_table_init(struct intel_vgpu *info) 433 { 434 hash_init(info->ptable); 435 } 436 437 static void kvmgt_protect_table_destroy(struct intel_vgpu *info) 438 { 439 struct kvmgt_pgfn *p; 440 struct hlist_node *tmp; 441 int i; 442 443 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 444 hash_del(&p->hnode); 445 kfree(p); 446 } 447 } 448 449 static struct kvmgt_pgfn * 450 __kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn) 451 { 452 struct kvmgt_pgfn *p, *res = NULL; 453 454 hash_for_each_possible(info->ptable, p, hnode, gfn) { 455 if (gfn == p->gfn) { 456 res = p; 457 break; 458 } 459 } 460 461 return res; 462 } 463 464 static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn) 465 { 466 struct kvmgt_pgfn *p; 467 468 p = __kvmgt_protect_table_find(info, gfn); 469 return !!p; 470 } 471 472 static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn) 473 { 474 struct kvmgt_pgfn *p; 475 476 if (kvmgt_gfn_is_write_protected(info, gfn)) 477 return; 478 479 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 480 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 481 return; 482 483 p->gfn = gfn; 484 hash_add(info->ptable, &p->hnode, gfn); 485 } 486 487 static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn) 488 { 489 struct kvmgt_pgfn *p; 490 491 p = __kvmgt_protect_table_find(info, gfn); 492 if (p) { 493 hash_del(&p->hnode); 494 kfree(p); 495 } 496 } 497 498 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, 499 size_t count, loff_t *ppos, bool iswrite) 500 { 501 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 502 VFIO_PCI_NUM_REGIONS; 503 void *base = vgpu->region[i].data; 504 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 505 506 507 if (pos >= vgpu->region[i].size || iswrite) { 508 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); 509 return -EINVAL; 510 } 511 count = min(count, (size_t)(vgpu->region[i].size - pos)); 512 memcpy(buf, base + pos, count); 513 514 return count; 515 } 516 517 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, 518 struct vfio_region *region) 519 { 520 } 521 522 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { 523 .rw = intel_vgpu_reg_rw_opregion, 524 .release = intel_vgpu_reg_release_opregion, 525 }; 526 527 static int handle_edid_regs(struct intel_vgpu *vgpu, 528 struct vfio_edid_region *region, char *buf, 529 size_t count, u16 offset, bool is_write) 530 { 531 struct vfio_region_gfx_edid *regs = ®ion->vfio_edid_regs; 532 unsigned int data; 533 534 if (offset + count > sizeof(*regs)) 535 return -EINVAL; 536 537 if (count != 4) 538 return -EINVAL; 539 540 if (is_write) { 541 data = *((unsigned int *)buf); 542 switch (offset) { 543 case offsetof(struct vfio_region_gfx_edid, link_state): 544 if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) { 545 if (!drm_edid_block_valid( 546 (u8 *)region->edid_blob, 547 0, 548 true, 549 NULL)) { 550 gvt_vgpu_err("invalid EDID blob\n"); 551 return -EINVAL; 552 } 553 intel_vgpu_emulate_hotplug(vgpu, true); 554 } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN) 555 intel_vgpu_emulate_hotplug(vgpu, false); 556 else { 557 gvt_vgpu_err("invalid EDID link state %d\n", 558 regs->link_state); 559 return -EINVAL; 560 } 561 regs->link_state = data; 562 break; 563 case offsetof(struct vfio_region_gfx_edid, edid_size): 564 if (data > regs->edid_max_size) { 565 gvt_vgpu_err("EDID size is bigger than %d!\n", 566 regs->edid_max_size); 567 return -EINVAL; 568 } 569 regs->edid_size = data; 570 break; 571 default: 572 /* read-only regs */ 573 gvt_vgpu_err("write read-only EDID region at offset %d\n", 574 offset); 575 return -EPERM; 576 } 577 } else { 578 memcpy(buf, (char *)regs + offset, count); 579 } 580 581 return count; 582 } 583 584 static int handle_edid_blob(struct vfio_edid_region *region, char *buf, 585 size_t count, u16 offset, bool is_write) 586 { 587 if (offset + count > region->vfio_edid_regs.edid_size) 588 return -EINVAL; 589 590 if (is_write) 591 memcpy(region->edid_blob + offset, buf, count); 592 else 593 memcpy(buf, region->edid_blob + offset, count); 594 595 return count; 596 } 597 598 static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf, 599 size_t count, loff_t *ppos, bool iswrite) 600 { 601 int ret; 602 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 603 VFIO_PCI_NUM_REGIONS; 604 struct vfio_edid_region *region = vgpu->region[i].data; 605 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 606 607 if (pos < region->vfio_edid_regs.edid_offset) { 608 ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite); 609 } else { 610 pos -= EDID_BLOB_OFFSET; 611 ret = handle_edid_blob(region, buf, count, pos, iswrite); 612 } 613 614 if (ret < 0) 615 gvt_vgpu_err("failed to access EDID region\n"); 616 617 return ret; 618 } 619 620 static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu, 621 struct vfio_region *region) 622 { 623 kfree(region->data); 624 } 625 626 static const struct intel_vgpu_regops intel_vgpu_regops_edid = { 627 .rw = intel_vgpu_reg_rw_edid, 628 .release = intel_vgpu_reg_release_edid, 629 }; 630 631 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, 632 unsigned int type, unsigned int subtype, 633 const struct intel_vgpu_regops *ops, 634 size_t size, u32 flags, void *data) 635 { 636 struct vfio_region *region; 637 638 region = krealloc(vgpu->region, 639 (vgpu->num_regions + 1) * sizeof(*region), 640 GFP_KERNEL); 641 if (!region) 642 return -ENOMEM; 643 644 vgpu->region = region; 645 vgpu->region[vgpu->num_regions].type = type; 646 vgpu->region[vgpu->num_regions].subtype = subtype; 647 vgpu->region[vgpu->num_regions].ops = ops; 648 vgpu->region[vgpu->num_regions].size = size; 649 vgpu->region[vgpu->num_regions].flags = flags; 650 vgpu->region[vgpu->num_regions].data = data; 651 vgpu->num_regions++; 652 return 0; 653 } 654 655 int intel_gvt_set_opregion(struct intel_vgpu *vgpu) 656 { 657 void *base; 658 int ret; 659 660 /* Each vgpu has its own opregion, although VFIO would create another 661 * one later. This one is used to expose opregion to VFIO. And the 662 * other one created by VFIO later, is used by guest actually. 663 */ 664 base = vgpu_opregion(vgpu)->va; 665 if (!base) 666 return -ENOMEM; 667 668 if (memcmp(base, OPREGION_SIGNATURE, 16)) { 669 memunmap(base); 670 return -EINVAL; 671 } 672 673 ret = intel_vgpu_register_reg(vgpu, 674 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, 675 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, 676 &intel_vgpu_regops_opregion, OPREGION_SIZE, 677 VFIO_REGION_INFO_FLAG_READ, base); 678 679 return ret; 680 } 681 682 int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num) 683 { 684 struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); 685 struct vfio_edid_region *base; 686 int ret; 687 688 base = kzalloc(sizeof(*base), GFP_KERNEL); 689 if (!base) 690 return -ENOMEM; 691 692 /* TODO: Add multi-port and EDID extension block support */ 693 base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET; 694 base->vfio_edid_regs.edid_max_size = EDID_SIZE; 695 base->vfio_edid_regs.edid_size = EDID_SIZE; 696 base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id); 697 base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id); 698 base->edid_blob = port->edid->edid_block; 699 700 ret = intel_vgpu_register_reg(vgpu, 701 VFIO_REGION_TYPE_GFX, 702 VFIO_REGION_SUBTYPE_GFX_EDID, 703 &intel_vgpu_regops_edid, EDID_SIZE, 704 VFIO_REGION_INFO_FLAG_READ | 705 VFIO_REGION_INFO_FLAG_WRITE | 706 VFIO_REGION_INFO_FLAG_CAPS, base); 707 708 return ret; 709 } 710 711 static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova, 712 u64 length) 713 { 714 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 715 struct gvt_dma *entry; 716 u64 iov_pfn = iova >> PAGE_SHIFT; 717 u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE; 718 719 mutex_lock(&vgpu->cache_lock); 720 for (; iov_pfn < end_iov_pfn; iov_pfn++) { 721 entry = __gvt_cache_find_gfn(vgpu, iov_pfn); 722 if (!entry) 723 continue; 724 725 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr, 726 entry->size); 727 __gvt_cache_remove_entry(vgpu, entry); 728 } 729 mutex_unlock(&vgpu->cache_lock); 730 } 731 732 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu) 733 { 734 struct intel_vgpu *itr; 735 int id; 736 bool ret = false; 737 738 mutex_lock(&vgpu->gvt->lock); 739 for_each_active_vgpu(vgpu->gvt, itr, id) { 740 if (!itr->attached) 741 continue; 742 743 if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) { 744 ret = true; 745 goto out; 746 } 747 } 748 out: 749 mutex_unlock(&vgpu->gvt->lock); 750 return ret; 751 } 752 753 static int intel_vgpu_open_device(struct vfio_device *vfio_dev) 754 { 755 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 756 757 if (vgpu->attached) 758 return -EEXIST; 759 760 if (!vgpu->vfio_device.kvm || 761 vgpu->vfio_device.kvm->mm != current->mm) { 762 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 763 return -ESRCH; 764 } 765 766 kvm_get_kvm(vgpu->vfio_device.kvm); 767 768 if (__kvmgt_vgpu_exist(vgpu)) 769 return -EEXIST; 770 771 vgpu->attached = true; 772 773 kvmgt_protect_table_init(vgpu); 774 gvt_cache_init(vgpu); 775 776 vgpu->track_node.track_write = kvmgt_page_track_write; 777 vgpu->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 778 kvm_page_track_register_notifier(vgpu->vfio_device.kvm, 779 &vgpu->track_node); 780 781 debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, 0444, vgpu->debugfs, 782 &vgpu->nr_cache_entries); 783 784 intel_gvt_activate_vgpu(vgpu); 785 786 atomic_set(&vgpu->released, 0); 787 return 0; 788 } 789 790 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu) 791 { 792 struct eventfd_ctx *trigger; 793 794 trigger = vgpu->msi_trigger; 795 if (trigger) { 796 eventfd_ctx_put(trigger); 797 vgpu->msi_trigger = NULL; 798 } 799 } 800 801 static void intel_vgpu_close_device(struct vfio_device *vfio_dev) 802 { 803 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 804 805 if (!vgpu->attached) 806 return; 807 808 if (atomic_cmpxchg(&vgpu->released, 0, 1)) 809 return; 810 811 intel_gvt_release_vgpu(vgpu); 812 813 debugfs_remove(debugfs_lookup(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs)); 814 815 kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm, 816 &vgpu->track_node); 817 kvmgt_protect_table_destroy(vgpu); 818 gvt_cache_destroy(vgpu); 819 820 intel_vgpu_release_msi_eventfd_ctx(vgpu); 821 822 vgpu->attached = false; 823 824 if (vgpu->vfio_device.kvm) 825 kvm_put_kvm(vgpu->vfio_device.kvm); 826 } 827 828 static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) 829 { 830 u32 start_lo, start_hi; 831 u32 mem_type; 832 833 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 834 PCI_BASE_ADDRESS_MEM_MASK; 835 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 836 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 837 838 switch (mem_type) { 839 case PCI_BASE_ADDRESS_MEM_TYPE_64: 840 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 841 + bar + 4)); 842 break; 843 case PCI_BASE_ADDRESS_MEM_TYPE_32: 844 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 845 /* 1M mem BAR treated as 32-bit BAR */ 846 default: 847 /* mem unknown type treated as 32-bit BAR */ 848 start_hi = 0; 849 break; 850 } 851 852 return ((u64)start_hi << 32) | start_lo; 853 } 854 855 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off, 856 void *buf, unsigned int count, bool is_write) 857 { 858 u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar); 859 int ret; 860 861 if (is_write) 862 ret = intel_vgpu_emulate_mmio_write(vgpu, 863 bar_start + off, buf, count); 864 else 865 ret = intel_vgpu_emulate_mmio_read(vgpu, 866 bar_start + off, buf, count); 867 return ret; 868 } 869 870 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off) 871 { 872 return off >= vgpu_aperture_offset(vgpu) && 873 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); 874 } 875 876 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off, 877 void *buf, unsigned long count, bool is_write) 878 { 879 void __iomem *aperture_va; 880 881 if (!intel_vgpu_in_aperture(vgpu, off) || 882 !intel_vgpu_in_aperture(vgpu, off + count)) { 883 gvt_vgpu_err("Invalid aperture offset %llu\n", off); 884 return -EINVAL; 885 } 886 887 aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap, 888 ALIGN_DOWN(off, PAGE_SIZE), 889 count + offset_in_page(off)); 890 if (!aperture_va) 891 return -EIO; 892 893 if (is_write) 894 memcpy_toio(aperture_va + offset_in_page(off), buf, count); 895 else 896 memcpy_fromio(buf, aperture_va + offset_in_page(off), count); 897 898 io_mapping_unmap(aperture_va); 899 900 return 0; 901 } 902 903 static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf, 904 size_t count, loff_t *ppos, bool is_write) 905 { 906 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 907 u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; 908 int ret = -EINVAL; 909 910 911 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) { 912 gvt_vgpu_err("invalid index: %u\n", index); 913 return -EINVAL; 914 } 915 916 switch (index) { 917 case VFIO_PCI_CONFIG_REGION_INDEX: 918 if (is_write) 919 ret = intel_vgpu_emulate_cfg_write(vgpu, pos, 920 buf, count); 921 else 922 ret = intel_vgpu_emulate_cfg_read(vgpu, pos, 923 buf, count); 924 break; 925 case VFIO_PCI_BAR0_REGION_INDEX: 926 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, 927 buf, count, is_write); 928 break; 929 case VFIO_PCI_BAR2_REGION_INDEX: 930 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); 931 break; 932 case VFIO_PCI_BAR1_REGION_INDEX: 933 case VFIO_PCI_BAR3_REGION_INDEX: 934 case VFIO_PCI_BAR4_REGION_INDEX: 935 case VFIO_PCI_BAR5_REGION_INDEX: 936 case VFIO_PCI_VGA_REGION_INDEX: 937 case VFIO_PCI_ROM_REGION_INDEX: 938 break; 939 default: 940 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) 941 return -EINVAL; 942 943 index -= VFIO_PCI_NUM_REGIONS; 944 return vgpu->region[index].ops->rw(vgpu, buf, count, 945 ppos, is_write); 946 } 947 948 return ret == 0 ? count : ret; 949 } 950 951 static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos) 952 { 953 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 954 struct intel_gvt *gvt = vgpu->gvt; 955 int offset; 956 957 /* Only allow MMIO GGTT entry access */ 958 if (index != PCI_BASE_ADDRESS_0) 959 return false; 960 961 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) - 962 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); 963 964 return (offset >= gvt->device_info.gtt_start_offset && 965 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ? 966 true : false; 967 } 968 969 static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf, 970 size_t count, loff_t *ppos) 971 { 972 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 973 unsigned int done = 0; 974 int ret; 975 976 while (count) { 977 size_t filled; 978 979 /* Only support GGTT entry 8 bytes read */ 980 if (count >= 8 && !(*ppos % 8) && 981 gtt_entry(vgpu, ppos)) { 982 u64 val; 983 984 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 985 ppos, false); 986 if (ret <= 0) 987 goto read_err; 988 989 if (copy_to_user(buf, &val, sizeof(val))) 990 goto read_err; 991 992 filled = 8; 993 } else if (count >= 4 && !(*ppos % 4)) { 994 u32 val; 995 996 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 997 ppos, false); 998 if (ret <= 0) 999 goto read_err; 1000 1001 if (copy_to_user(buf, &val, sizeof(val))) 1002 goto read_err; 1003 1004 filled = 4; 1005 } else if (count >= 2 && !(*ppos % 2)) { 1006 u16 val; 1007 1008 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 1009 ppos, false); 1010 if (ret <= 0) 1011 goto read_err; 1012 1013 if (copy_to_user(buf, &val, sizeof(val))) 1014 goto read_err; 1015 1016 filled = 2; 1017 } else { 1018 u8 val; 1019 1020 ret = intel_vgpu_rw(vgpu, &val, sizeof(val), ppos, 1021 false); 1022 if (ret <= 0) 1023 goto read_err; 1024 1025 if (copy_to_user(buf, &val, sizeof(val))) 1026 goto read_err; 1027 1028 filled = 1; 1029 } 1030 1031 count -= filled; 1032 done += filled; 1033 *ppos += filled; 1034 buf += filled; 1035 } 1036 1037 return done; 1038 1039 read_err: 1040 return -EFAULT; 1041 } 1042 1043 static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev, 1044 const char __user *buf, 1045 size_t count, loff_t *ppos) 1046 { 1047 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1048 unsigned int done = 0; 1049 int ret; 1050 1051 while (count) { 1052 size_t filled; 1053 1054 /* Only support GGTT entry 8 bytes write */ 1055 if (count >= 8 && !(*ppos % 8) && 1056 gtt_entry(vgpu, ppos)) { 1057 u64 val; 1058 1059 if (copy_from_user(&val, buf, sizeof(val))) 1060 goto write_err; 1061 1062 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 1063 ppos, true); 1064 if (ret <= 0) 1065 goto write_err; 1066 1067 filled = 8; 1068 } else if (count >= 4 && !(*ppos % 4)) { 1069 u32 val; 1070 1071 if (copy_from_user(&val, buf, sizeof(val))) 1072 goto write_err; 1073 1074 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 1075 ppos, true); 1076 if (ret <= 0) 1077 goto write_err; 1078 1079 filled = 4; 1080 } else if (count >= 2 && !(*ppos % 2)) { 1081 u16 val; 1082 1083 if (copy_from_user(&val, buf, sizeof(val))) 1084 goto write_err; 1085 1086 ret = intel_vgpu_rw(vgpu, (char *)&val, 1087 sizeof(val), ppos, true); 1088 if (ret <= 0) 1089 goto write_err; 1090 1091 filled = 2; 1092 } else { 1093 u8 val; 1094 1095 if (copy_from_user(&val, buf, sizeof(val))) 1096 goto write_err; 1097 1098 ret = intel_vgpu_rw(vgpu, &val, sizeof(val), 1099 ppos, true); 1100 if (ret <= 0) 1101 goto write_err; 1102 1103 filled = 1; 1104 } 1105 1106 count -= filled; 1107 done += filled; 1108 *ppos += filled; 1109 buf += filled; 1110 } 1111 1112 return done; 1113 write_err: 1114 return -EFAULT; 1115 } 1116 1117 static int intel_vgpu_mmap(struct vfio_device *vfio_dev, 1118 struct vm_area_struct *vma) 1119 { 1120 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1121 unsigned int index; 1122 u64 virtaddr; 1123 unsigned long req_size, pgoff, req_start; 1124 pgprot_t pg_prot; 1125 1126 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1127 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1128 return -EINVAL; 1129 1130 if (vma->vm_end < vma->vm_start) 1131 return -EINVAL; 1132 if ((vma->vm_flags & VM_SHARED) == 0) 1133 return -EINVAL; 1134 if (index != VFIO_PCI_BAR2_REGION_INDEX) 1135 return -EINVAL; 1136 1137 pg_prot = vma->vm_page_prot; 1138 virtaddr = vma->vm_start; 1139 req_size = vma->vm_end - vma->vm_start; 1140 pgoff = vma->vm_pgoff & 1141 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1142 req_start = pgoff << PAGE_SHIFT; 1143 1144 if (!intel_vgpu_in_aperture(vgpu, req_start)) 1145 return -EINVAL; 1146 if (req_start + req_size > 1147 vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu)) 1148 return -EINVAL; 1149 1150 pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff; 1151 1152 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 1153 } 1154 1155 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 1156 { 1157 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 1158 return 1; 1159 1160 return 0; 1161 } 1162 1163 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 1164 unsigned int index, unsigned int start, 1165 unsigned int count, u32 flags, 1166 void *data) 1167 { 1168 return 0; 1169 } 1170 1171 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 1172 unsigned int index, unsigned int start, 1173 unsigned int count, u32 flags, void *data) 1174 { 1175 return 0; 1176 } 1177 1178 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 1179 unsigned int index, unsigned int start, unsigned int count, 1180 u32 flags, void *data) 1181 { 1182 return 0; 1183 } 1184 1185 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 1186 unsigned int index, unsigned int start, unsigned int count, 1187 u32 flags, void *data) 1188 { 1189 struct eventfd_ctx *trigger; 1190 1191 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1192 int fd = *(int *)data; 1193 1194 trigger = eventfd_ctx_fdget(fd); 1195 if (IS_ERR(trigger)) { 1196 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 1197 return PTR_ERR(trigger); 1198 } 1199 vgpu->msi_trigger = trigger; 1200 } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count) 1201 intel_vgpu_release_msi_eventfd_ctx(vgpu); 1202 1203 return 0; 1204 } 1205 1206 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags, 1207 unsigned int index, unsigned int start, unsigned int count, 1208 void *data) 1209 { 1210 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 1211 unsigned int start, unsigned int count, u32 flags, 1212 void *data) = NULL; 1213 1214 switch (index) { 1215 case VFIO_PCI_INTX_IRQ_INDEX: 1216 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1217 case VFIO_IRQ_SET_ACTION_MASK: 1218 func = intel_vgpu_set_intx_mask; 1219 break; 1220 case VFIO_IRQ_SET_ACTION_UNMASK: 1221 func = intel_vgpu_set_intx_unmask; 1222 break; 1223 case VFIO_IRQ_SET_ACTION_TRIGGER: 1224 func = intel_vgpu_set_intx_trigger; 1225 break; 1226 } 1227 break; 1228 case VFIO_PCI_MSI_IRQ_INDEX: 1229 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1230 case VFIO_IRQ_SET_ACTION_MASK: 1231 case VFIO_IRQ_SET_ACTION_UNMASK: 1232 /* XXX Need masking support exported */ 1233 break; 1234 case VFIO_IRQ_SET_ACTION_TRIGGER: 1235 func = intel_vgpu_set_msi_trigger; 1236 break; 1237 } 1238 break; 1239 } 1240 1241 if (!func) 1242 return -ENOTTY; 1243 1244 return func(vgpu, index, start, count, flags, data); 1245 } 1246 1247 static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd, 1248 unsigned long arg) 1249 { 1250 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1251 unsigned long minsz; 1252 1253 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 1254 1255 if (cmd == VFIO_DEVICE_GET_INFO) { 1256 struct vfio_device_info info; 1257 1258 minsz = offsetofend(struct vfio_device_info, num_irqs); 1259 1260 if (copy_from_user(&info, (void __user *)arg, minsz)) 1261 return -EFAULT; 1262 1263 if (info.argsz < minsz) 1264 return -EINVAL; 1265 1266 info.flags = VFIO_DEVICE_FLAGS_PCI; 1267 info.flags |= VFIO_DEVICE_FLAGS_RESET; 1268 info.num_regions = VFIO_PCI_NUM_REGIONS + 1269 vgpu->num_regions; 1270 info.num_irqs = VFIO_PCI_NUM_IRQS; 1271 1272 return copy_to_user((void __user *)arg, &info, minsz) ? 1273 -EFAULT : 0; 1274 1275 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 1276 struct vfio_region_info info; 1277 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 1278 unsigned int i; 1279 int ret; 1280 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 1281 int nr_areas = 1; 1282 int cap_type_id; 1283 1284 minsz = offsetofend(struct vfio_region_info, offset); 1285 1286 if (copy_from_user(&info, (void __user *)arg, minsz)) 1287 return -EFAULT; 1288 1289 if (info.argsz < minsz) 1290 return -EINVAL; 1291 1292 switch (info.index) { 1293 case VFIO_PCI_CONFIG_REGION_INDEX: 1294 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1295 info.size = vgpu->gvt->device_info.cfg_space_size; 1296 info.flags = VFIO_REGION_INFO_FLAG_READ | 1297 VFIO_REGION_INFO_FLAG_WRITE; 1298 break; 1299 case VFIO_PCI_BAR0_REGION_INDEX: 1300 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1301 info.size = vgpu->cfg_space.bar[info.index].size; 1302 if (!info.size) { 1303 info.flags = 0; 1304 break; 1305 } 1306 1307 info.flags = VFIO_REGION_INFO_FLAG_READ | 1308 VFIO_REGION_INFO_FLAG_WRITE; 1309 break; 1310 case VFIO_PCI_BAR1_REGION_INDEX: 1311 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1312 info.size = 0; 1313 info.flags = 0; 1314 break; 1315 case VFIO_PCI_BAR2_REGION_INDEX: 1316 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1317 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 1318 VFIO_REGION_INFO_FLAG_MMAP | 1319 VFIO_REGION_INFO_FLAG_READ | 1320 VFIO_REGION_INFO_FLAG_WRITE; 1321 info.size = gvt_aperture_sz(vgpu->gvt); 1322 1323 sparse = kzalloc(struct_size(sparse, areas, nr_areas), 1324 GFP_KERNEL); 1325 if (!sparse) 1326 return -ENOMEM; 1327 1328 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1329 sparse->header.version = 1; 1330 sparse->nr_areas = nr_areas; 1331 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1332 sparse->areas[0].offset = 1333 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1334 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1335 break; 1336 1337 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1338 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1339 info.size = 0; 1340 info.flags = 0; 1341 1342 gvt_dbg_core("get region info bar:%d\n", info.index); 1343 break; 1344 1345 case VFIO_PCI_ROM_REGION_INDEX: 1346 case VFIO_PCI_VGA_REGION_INDEX: 1347 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1348 info.size = 0; 1349 info.flags = 0; 1350 1351 gvt_dbg_core("get region info index:%d\n", info.index); 1352 break; 1353 default: 1354 { 1355 struct vfio_region_info_cap_type cap_type = { 1356 .header.id = VFIO_REGION_INFO_CAP_TYPE, 1357 .header.version = 1 }; 1358 1359 if (info.index >= VFIO_PCI_NUM_REGIONS + 1360 vgpu->num_regions) 1361 return -EINVAL; 1362 info.index = 1363 array_index_nospec(info.index, 1364 VFIO_PCI_NUM_REGIONS + 1365 vgpu->num_regions); 1366 1367 i = info.index - VFIO_PCI_NUM_REGIONS; 1368 1369 info.offset = 1370 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1371 info.size = vgpu->region[i].size; 1372 info.flags = vgpu->region[i].flags; 1373 1374 cap_type.type = vgpu->region[i].type; 1375 cap_type.subtype = vgpu->region[i].subtype; 1376 1377 ret = vfio_info_add_capability(&caps, 1378 &cap_type.header, 1379 sizeof(cap_type)); 1380 if (ret) 1381 return ret; 1382 } 1383 } 1384 1385 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1386 switch (cap_type_id) { 1387 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1388 ret = vfio_info_add_capability(&caps, 1389 &sparse->header, 1390 struct_size(sparse, areas, 1391 sparse->nr_areas)); 1392 if (ret) { 1393 kfree(sparse); 1394 return ret; 1395 } 1396 break; 1397 default: 1398 kfree(sparse); 1399 return -EINVAL; 1400 } 1401 } 1402 1403 if (caps.size) { 1404 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1405 if (info.argsz < sizeof(info) + caps.size) { 1406 info.argsz = sizeof(info) + caps.size; 1407 info.cap_offset = 0; 1408 } else { 1409 vfio_info_cap_shift(&caps, sizeof(info)); 1410 if (copy_to_user((void __user *)arg + 1411 sizeof(info), caps.buf, 1412 caps.size)) { 1413 kfree(caps.buf); 1414 kfree(sparse); 1415 return -EFAULT; 1416 } 1417 info.cap_offset = sizeof(info); 1418 } 1419 1420 kfree(caps.buf); 1421 } 1422 1423 kfree(sparse); 1424 return copy_to_user((void __user *)arg, &info, minsz) ? 1425 -EFAULT : 0; 1426 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1427 struct vfio_irq_info info; 1428 1429 minsz = offsetofend(struct vfio_irq_info, count); 1430 1431 if (copy_from_user(&info, (void __user *)arg, minsz)) 1432 return -EFAULT; 1433 1434 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1435 return -EINVAL; 1436 1437 switch (info.index) { 1438 case VFIO_PCI_INTX_IRQ_INDEX: 1439 case VFIO_PCI_MSI_IRQ_INDEX: 1440 break; 1441 default: 1442 return -EINVAL; 1443 } 1444 1445 info.flags = VFIO_IRQ_INFO_EVENTFD; 1446 1447 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1448 1449 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1450 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1451 VFIO_IRQ_INFO_AUTOMASKED); 1452 else 1453 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1454 1455 return copy_to_user((void __user *)arg, &info, minsz) ? 1456 -EFAULT : 0; 1457 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1458 struct vfio_irq_set hdr; 1459 u8 *data = NULL; 1460 int ret = 0; 1461 size_t data_size = 0; 1462 1463 minsz = offsetofend(struct vfio_irq_set, count); 1464 1465 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1466 return -EFAULT; 1467 1468 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1469 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1470 1471 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1472 VFIO_PCI_NUM_IRQS, &data_size); 1473 if (ret) { 1474 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1475 return -EINVAL; 1476 } 1477 if (data_size) { 1478 data = memdup_user((void __user *)(arg + minsz), 1479 data_size); 1480 if (IS_ERR(data)) 1481 return PTR_ERR(data); 1482 } 1483 } 1484 1485 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1486 hdr.start, hdr.count, data); 1487 kfree(data); 1488 1489 return ret; 1490 } else if (cmd == VFIO_DEVICE_RESET) { 1491 intel_gvt_reset_vgpu(vgpu); 1492 return 0; 1493 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { 1494 struct vfio_device_gfx_plane_info dmabuf; 1495 int ret = 0; 1496 1497 minsz = offsetofend(struct vfio_device_gfx_plane_info, 1498 dmabuf_id); 1499 if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) 1500 return -EFAULT; 1501 if (dmabuf.argsz < minsz) 1502 return -EINVAL; 1503 1504 ret = intel_vgpu_query_plane(vgpu, &dmabuf); 1505 if (ret != 0) 1506 return ret; 1507 1508 return copy_to_user((void __user *)arg, &dmabuf, minsz) ? 1509 -EFAULT : 0; 1510 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { 1511 __u32 dmabuf_id; 1512 1513 if (get_user(dmabuf_id, (__u32 __user *)arg)) 1514 return -EFAULT; 1515 return intel_vgpu_get_dmabuf(vgpu, dmabuf_id); 1516 } 1517 1518 return -ENOTTY; 1519 } 1520 1521 static ssize_t 1522 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1523 char *buf) 1524 { 1525 struct intel_vgpu *vgpu = dev_get_drvdata(dev); 1526 1527 return sprintf(buf, "%d\n", vgpu->id); 1528 } 1529 1530 static DEVICE_ATTR_RO(vgpu_id); 1531 1532 static struct attribute *intel_vgpu_attrs[] = { 1533 &dev_attr_vgpu_id.attr, 1534 NULL 1535 }; 1536 1537 static const struct attribute_group intel_vgpu_group = { 1538 .name = "intel_vgpu", 1539 .attrs = intel_vgpu_attrs, 1540 }; 1541 1542 static const struct attribute_group *intel_vgpu_groups[] = { 1543 &intel_vgpu_group, 1544 NULL, 1545 }; 1546 1547 static int intel_vgpu_init_dev(struct vfio_device *vfio_dev) 1548 { 1549 struct mdev_device *mdev = to_mdev_device(vfio_dev->dev); 1550 struct device *pdev = mdev_parent_dev(mdev); 1551 struct intel_gvt *gvt = kdev_to_i915(pdev)->gvt; 1552 struct intel_vgpu_type *type; 1553 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1554 1555 type = &gvt->types[mdev_get_type_group_id(mdev)]; 1556 if (!type) 1557 return -EINVAL; 1558 1559 vgpu->gvt = gvt; 1560 return intel_gvt_create_vgpu(vgpu, type->conf); 1561 } 1562 1563 static void intel_vgpu_release_dev(struct vfio_device *vfio_dev) 1564 { 1565 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1566 1567 intel_gvt_destroy_vgpu(vgpu); 1568 vfio_free_device(vfio_dev); 1569 } 1570 1571 static const struct vfio_device_ops intel_vgpu_dev_ops = { 1572 .init = intel_vgpu_init_dev, 1573 .release = intel_vgpu_release_dev, 1574 .open_device = intel_vgpu_open_device, 1575 .close_device = intel_vgpu_close_device, 1576 .read = intel_vgpu_read, 1577 .write = intel_vgpu_write, 1578 .mmap = intel_vgpu_mmap, 1579 .ioctl = intel_vgpu_ioctl, 1580 .dma_unmap = intel_vgpu_dma_unmap, 1581 }; 1582 1583 static int intel_vgpu_probe(struct mdev_device *mdev) 1584 { 1585 struct intel_vgpu *vgpu; 1586 int ret; 1587 1588 vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev, 1589 &intel_vgpu_dev_ops); 1590 if (IS_ERR(vgpu)) { 1591 gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu)); 1592 return PTR_ERR(vgpu); 1593 } 1594 1595 dev_set_drvdata(&mdev->dev, vgpu); 1596 ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device); 1597 if (ret) 1598 goto out_put_vdev; 1599 1600 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 1601 dev_name(mdev_dev(mdev))); 1602 return 0; 1603 1604 out_put_vdev: 1605 vfio_put_device(&vgpu->vfio_device); 1606 return ret; 1607 } 1608 1609 static void intel_vgpu_remove(struct mdev_device *mdev) 1610 { 1611 struct intel_vgpu *vgpu = dev_get_drvdata(&mdev->dev); 1612 1613 if (WARN_ON_ONCE(vgpu->attached)) 1614 return; 1615 1616 vfio_unregister_group_dev(&vgpu->vfio_device); 1617 vfio_put_device(&vgpu->vfio_device); 1618 } 1619 1620 static struct mdev_driver intel_vgpu_mdev_driver = { 1621 .driver = { 1622 .name = "intel_vgpu_mdev", 1623 .owner = THIS_MODULE, 1624 .dev_groups = intel_vgpu_groups, 1625 }, 1626 .probe = intel_vgpu_probe, 1627 .remove = intel_vgpu_remove, 1628 .supported_type_groups = gvt_vgpu_type_groups, 1629 }; 1630 1631 int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn) 1632 { 1633 struct kvm *kvm = info->vfio_device.kvm; 1634 struct kvm_memory_slot *slot; 1635 int idx; 1636 1637 if (!info->attached) 1638 return -ESRCH; 1639 1640 idx = srcu_read_lock(&kvm->srcu); 1641 slot = gfn_to_memslot(kvm, gfn); 1642 if (!slot) { 1643 srcu_read_unlock(&kvm->srcu, idx); 1644 return -EINVAL; 1645 } 1646 1647 write_lock(&kvm->mmu_lock); 1648 1649 if (kvmgt_gfn_is_write_protected(info, gfn)) 1650 goto out; 1651 1652 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1653 kvmgt_protect_table_add(info, gfn); 1654 1655 out: 1656 write_unlock(&kvm->mmu_lock); 1657 srcu_read_unlock(&kvm->srcu, idx); 1658 return 0; 1659 } 1660 1661 int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn) 1662 { 1663 struct kvm *kvm = info->vfio_device.kvm; 1664 struct kvm_memory_slot *slot; 1665 int idx; 1666 1667 if (!info->attached) 1668 return 0; 1669 1670 idx = srcu_read_lock(&kvm->srcu); 1671 slot = gfn_to_memslot(kvm, gfn); 1672 if (!slot) { 1673 srcu_read_unlock(&kvm->srcu, idx); 1674 return -EINVAL; 1675 } 1676 1677 write_lock(&kvm->mmu_lock); 1678 1679 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1680 goto out; 1681 1682 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1683 kvmgt_protect_table_del(info, gfn); 1684 1685 out: 1686 write_unlock(&kvm->mmu_lock); 1687 srcu_read_unlock(&kvm->srcu, idx); 1688 return 0; 1689 } 1690 1691 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1692 const u8 *val, int len, 1693 struct kvm_page_track_notifier_node *node) 1694 { 1695 struct intel_vgpu *info = 1696 container_of(node, struct intel_vgpu, track_node); 1697 1698 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1699 intel_vgpu_page_track_handler(info, gpa, 1700 (void *)val, len); 1701 } 1702 1703 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1704 struct kvm_memory_slot *slot, 1705 struct kvm_page_track_notifier_node *node) 1706 { 1707 int i; 1708 gfn_t gfn; 1709 struct intel_vgpu *info = 1710 container_of(node, struct intel_vgpu, track_node); 1711 1712 write_lock(&kvm->mmu_lock); 1713 for (i = 0; i < slot->npages; i++) { 1714 gfn = slot->base_gfn + i; 1715 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1716 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1717 KVM_PAGE_TRACK_WRITE); 1718 kvmgt_protect_table_del(info, gfn); 1719 } 1720 } 1721 write_unlock(&kvm->mmu_lock); 1722 } 1723 1724 void intel_vgpu_detach_regions(struct intel_vgpu *vgpu) 1725 { 1726 int i; 1727 1728 if (!vgpu->region) 1729 return; 1730 1731 for (i = 0; i < vgpu->num_regions; i++) 1732 if (vgpu->region[i].ops->release) 1733 vgpu->region[i].ops->release(vgpu, 1734 &vgpu->region[i]); 1735 vgpu->num_regions = 0; 1736 kfree(vgpu->region); 1737 vgpu->region = NULL; 1738 } 1739 1740 int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 1741 unsigned long size, dma_addr_t *dma_addr) 1742 { 1743 struct gvt_dma *entry; 1744 int ret; 1745 1746 if (!vgpu->attached) 1747 return -EINVAL; 1748 1749 mutex_lock(&vgpu->cache_lock); 1750 1751 entry = __gvt_cache_find_gfn(vgpu, gfn); 1752 if (!entry) { 1753 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); 1754 if (ret) 1755 goto err_unlock; 1756 1757 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size); 1758 if (ret) 1759 goto err_unmap; 1760 } else if (entry->size != size) { 1761 /* the same gfn with different size: unmap and re-map */ 1762 gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size); 1763 __gvt_cache_remove_entry(vgpu, entry); 1764 1765 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); 1766 if (ret) 1767 goto err_unlock; 1768 1769 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size); 1770 if (ret) 1771 goto err_unmap; 1772 } else { 1773 kref_get(&entry->ref); 1774 *dma_addr = entry->dma_addr; 1775 } 1776 1777 mutex_unlock(&vgpu->cache_lock); 1778 return 0; 1779 1780 err_unmap: 1781 gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size); 1782 err_unlock: 1783 mutex_unlock(&vgpu->cache_lock); 1784 return ret; 1785 } 1786 1787 int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr) 1788 { 1789 struct gvt_dma *entry; 1790 int ret = 0; 1791 1792 if (!vgpu->attached) 1793 return -ENODEV; 1794 1795 mutex_lock(&vgpu->cache_lock); 1796 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr); 1797 if (entry) 1798 kref_get(&entry->ref); 1799 else 1800 ret = -ENOMEM; 1801 mutex_unlock(&vgpu->cache_lock); 1802 1803 return ret; 1804 } 1805 1806 static void __gvt_dma_release(struct kref *ref) 1807 { 1808 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref); 1809 1810 gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr, 1811 entry->size); 1812 __gvt_cache_remove_entry(entry->vgpu, entry); 1813 } 1814 1815 void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu, 1816 dma_addr_t dma_addr) 1817 { 1818 struct gvt_dma *entry; 1819 1820 if (!vgpu->attached) 1821 return; 1822 1823 mutex_lock(&vgpu->cache_lock); 1824 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr); 1825 if (entry) 1826 kref_put(&entry->ref, __gvt_dma_release); 1827 mutex_unlock(&vgpu->cache_lock); 1828 } 1829 1830 static void init_device_info(struct intel_gvt *gvt) 1831 { 1832 struct intel_gvt_device_info *info = &gvt->device_info; 1833 struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev); 1834 1835 info->max_support_vgpus = 8; 1836 info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE; 1837 info->mmio_size = 2 * 1024 * 1024; 1838 info->mmio_bar = 0; 1839 info->gtt_start_offset = 8 * 1024 * 1024; 1840 info->gtt_entry_size = 8; 1841 info->gtt_entry_size_shift = 3; 1842 info->gmadr_bytes_in_cmd = 8; 1843 info->max_surface_size = 36 * 1024 * 1024; 1844 info->msi_cap_offset = pdev->msi_cap; 1845 } 1846 1847 static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt) 1848 { 1849 struct intel_vgpu *vgpu; 1850 int id; 1851 1852 mutex_lock(&gvt->lock); 1853 idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) { 1854 if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id, 1855 (void *)&gvt->service_request)) { 1856 if (vgpu->active) 1857 intel_vgpu_emulate_vblank(vgpu); 1858 } 1859 } 1860 mutex_unlock(&gvt->lock); 1861 } 1862 1863 static int gvt_service_thread(void *data) 1864 { 1865 struct intel_gvt *gvt = (struct intel_gvt *)data; 1866 int ret; 1867 1868 gvt_dbg_core("service thread start\n"); 1869 1870 while (!kthread_should_stop()) { 1871 ret = wait_event_interruptible(gvt->service_thread_wq, 1872 kthread_should_stop() || gvt->service_request); 1873 1874 if (kthread_should_stop()) 1875 break; 1876 1877 if (WARN_ONCE(ret, "service thread is waken up by signal.\n")) 1878 continue; 1879 1880 intel_gvt_test_and_emulate_vblank(gvt); 1881 1882 if (test_bit(INTEL_GVT_REQUEST_SCHED, 1883 (void *)&gvt->service_request) || 1884 test_bit(INTEL_GVT_REQUEST_EVENT_SCHED, 1885 (void *)&gvt->service_request)) { 1886 intel_gvt_schedule(gvt); 1887 } 1888 } 1889 1890 return 0; 1891 } 1892 1893 static void clean_service_thread(struct intel_gvt *gvt) 1894 { 1895 kthread_stop(gvt->service_thread); 1896 } 1897 1898 static int init_service_thread(struct intel_gvt *gvt) 1899 { 1900 init_waitqueue_head(&gvt->service_thread_wq); 1901 1902 gvt->service_thread = kthread_run(gvt_service_thread, 1903 gvt, "gvt_service_thread"); 1904 if (IS_ERR(gvt->service_thread)) { 1905 gvt_err("fail to start service thread.\n"); 1906 return PTR_ERR(gvt->service_thread); 1907 } 1908 return 0; 1909 } 1910 1911 /** 1912 * intel_gvt_clean_device - clean a GVT device 1913 * @i915: i915 private 1914 * 1915 * This function is called at the driver unloading stage, to free the 1916 * resources owned by a GVT device. 1917 * 1918 */ 1919 static void intel_gvt_clean_device(struct drm_i915_private *i915) 1920 { 1921 struct intel_gvt *gvt = fetch_and_zero(&i915->gvt); 1922 1923 if (drm_WARN_ON(&i915->drm, !gvt)) 1924 return; 1925 1926 mdev_unregister_parent(&gvt->parent); 1927 intel_gvt_cleanup_vgpu_type_groups(gvt); 1928 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); 1929 intel_gvt_clean_vgpu_types(gvt); 1930 1931 intel_gvt_debugfs_clean(gvt); 1932 clean_service_thread(gvt); 1933 intel_gvt_clean_cmd_parser(gvt); 1934 intel_gvt_clean_sched_policy(gvt); 1935 intel_gvt_clean_workload_scheduler(gvt); 1936 intel_gvt_clean_gtt(gvt); 1937 intel_gvt_free_firmware(gvt); 1938 intel_gvt_clean_mmio_info(gvt); 1939 idr_destroy(&gvt->vgpu_idr); 1940 1941 kfree(i915->gvt); 1942 } 1943 1944 /** 1945 * intel_gvt_init_device - initialize a GVT device 1946 * @i915: drm i915 private data 1947 * 1948 * This function is called at the initialization stage, to initialize 1949 * necessary GVT components. 1950 * 1951 * Returns: 1952 * Zero on success, negative error code if failed. 1953 * 1954 */ 1955 static int intel_gvt_init_device(struct drm_i915_private *i915) 1956 { 1957 struct intel_gvt *gvt; 1958 struct intel_vgpu *vgpu; 1959 int ret; 1960 1961 if (drm_WARN_ON(&i915->drm, i915->gvt)) 1962 return -EEXIST; 1963 1964 gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL); 1965 if (!gvt) 1966 return -ENOMEM; 1967 1968 gvt_dbg_core("init gvt device\n"); 1969 1970 idr_init_base(&gvt->vgpu_idr, 1); 1971 spin_lock_init(&gvt->scheduler.mmio_context_lock); 1972 mutex_init(&gvt->lock); 1973 mutex_init(&gvt->sched_lock); 1974 gvt->gt = to_gt(i915); 1975 i915->gvt = gvt; 1976 1977 init_device_info(gvt); 1978 1979 ret = intel_gvt_setup_mmio_info(gvt); 1980 if (ret) 1981 goto out_clean_idr; 1982 1983 intel_gvt_init_engine_mmio_context(gvt); 1984 1985 ret = intel_gvt_load_firmware(gvt); 1986 if (ret) 1987 goto out_clean_mmio_info; 1988 1989 ret = intel_gvt_init_irq(gvt); 1990 if (ret) 1991 goto out_free_firmware; 1992 1993 ret = intel_gvt_init_gtt(gvt); 1994 if (ret) 1995 goto out_free_firmware; 1996 1997 ret = intel_gvt_init_workload_scheduler(gvt); 1998 if (ret) 1999 goto out_clean_gtt; 2000 2001 ret = intel_gvt_init_sched_policy(gvt); 2002 if (ret) 2003 goto out_clean_workload_scheduler; 2004 2005 ret = intel_gvt_init_cmd_parser(gvt); 2006 if (ret) 2007 goto out_clean_sched_policy; 2008 2009 ret = init_service_thread(gvt); 2010 if (ret) 2011 goto out_clean_cmd_parser; 2012 2013 ret = intel_gvt_init_vgpu_types(gvt); 2014 if (ret) 2015 goto out_clean_thread; 2016 2017 vgpu = intel_gvt_create_idle_vgpu(gvt); 2018 if (IS_ERR(vgpu)) { 2019 ret = PTR_ERR(vgpu); 2020 gvt_err("failed to create idle vgpu\n"); 2021 goto out_clean_types; 2022 } 2023 gvt->idle_vgpu = vgpu; 2024 2025 intel_gvt_debugfs_init(gvt); 2026 2027 ret = intel_gvt_init_vgpu_type_groups(gvt); 2028 if (ret) 2029 goto out_destroy_idle_vgpu; 2030 2031 ret = mdev_register_parent(&gvt->parent, i915->drm.dev, 2032 &intel_vgpu_mdev_driver); 2033 if (ret) 2034 goto out_cleanup_vgpu_type_groups; 2035 2036 gvt_dbg_core("gvt device initialization is done\n"); 2037 return 0; 2038 2039 out_cleanup_vgpu_type_groups: 2040 intel_gvt_cleanup_vgpu_type_groups(gvt); 2041 out_destroy_idle_vgpu: 2042 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); 2043 intel_gvt_debugfs_clean(gvt); 2044 out_clean_types: 2045 intel_gvt_clean_vgpu_types(gvt); 2046 out_clean_thread: 2047 clean_service_thread(gvt); 2048 out_clean_cmd_parser: 2049 intel_gvt_clean_cmd_parser(gvt); 2050 out_clean_sched_policy: 2051 intel_gvt_clean_sched_policy(gvt); 2052 out_clean_workload_scheduler: 2053 intel_gvt_clean_workload_scheduler(gvt); 2054 out_clean_gtt: 2055 intel_gvt_clean_gtt(gvt); 2056 out_free_firmware: 2057 intel_gvt_free_firmware(gvt); 2058 out_clean_mmio_info: 2059 intel_gvt_clean_mmio_info(gvt); 2060 out_clean_idr: 2061 idr_destroy(&gvt->vgpu_idr); 2062 kfree(gvt); 2063 i915->gvt = NULL; 2064 return ret; 2065 } 2066 2067 static void intel_gvt_pm_resume(struct drm_i915_private *i915) 2068 { 2069 struct intel_gvt *gvt = i915->gvt; 2070 2071 intel_gvt_restore_fence(gvt); 2072 intel_gvt_restore_mmio(gvt); 2073 intel_gvt_restore_ggtt(gvt); 2074 } 2075 2076 static const struct intel_vgpu_ops intel_gvt_vgpu_ops = { 2077 .init_device = intel_gvt_init_device, 2078 .clean_device = intel_gvt_clean_device, 2079 .pm_resume = intel_gvt_pm_resume, 2080 }; 2081 2082 static int __init kvmgt_init(void) 2083 { 2084 int ret; 2085 2086 ret = intel_gvt_set_ops(&intel_gvt_vgpu_ops); 2087 if (ret) 2088 return ret; 2089 2090 ret = mdev_register_driver(&intel_vgpu_mdev_driver); 2091 if (ret) 2092 intel_gvt_clear_ops(&intel_gvt_vgpu_ops); 2093 return ret; 2094 } 2095 2096 static void __exit kvmgt_exit(void) 2097 { 2098 mdev_unregister_driver(&intel_vgpu_mdev_driver); 2099 intel_gvt_clear_ops(&intel_gvt_vgpu_ops); 2100 } 2101 2102 module_init(kvmgt_init); 2103 module_exit(kvmgt_exit); 2104 2105 MODULE_LICENSE("GPL and additional rights"); 2106 MODULE_AUTHOR("Intel Corporation"); 2107