1 /* 2 * Copyright 2017 Intel Corporation. All rights reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: 24 * Zhiyuan Lv <zhiyuan.lv@intel.com> 25 * 26 * Contributors: 27 * Xiaoguang Chen 28 * Tina Zhang <tina.zhang@intel.com> 29 */ 30 31 #include <linux/dma-buf.h> 32 #include <drm/drmP.h> 33 #include <linux/vfio.h> 34 35 #include "i915_drv.h" 36 #include "gvt.h" 37 38 #define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12)) 39 40 static int vgpu_gem_get_pages( 41 struct drm_i915_gem_object *obj) 42 { 43 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 44 struct sg_table *st; 45 struct scatterlist *sg; 46 int i, ret; 47 gen8_pte_t __iomem *gtt_entries; 48 struct intel_vgpu_fb_info *fb_info; 49 50 fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info; 51 if (WARN_ON(!fb_info)) 52 return -ENODEV; 53 54 st = kmalloc(sizeof(*st), GFP_KERNEL); 55 if (unlikely(!st)) 56 return -ENOMEM; 57 58 ret = sg_alloc_table(st, fb_info->size, GFP_KERNEL); 59 if (ret) { 60 kfree(st); 61 return ret; 62 } 63 gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + 64 (fb_info->start >> PAGE_SHIFT); 65 for_each_sg(st->sgl, sg, fb_info->size, i) { 66 sg->offset = 0; 67 sg->length = PAGE_SIZE; 68 sg_dma_address(sg) = 69 GEN8_DECODE_PTE(readq(>t_entries[i])); 70 sg_dma_len(sg) = PAGE_SIZE; 71 } 72 73 __i915_gem_object_set_pages(obj, st, PAGE_SIZE); 74 75 return 0; 76 } 77 78 static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj, 79 struct sg_table *pages) 80 { 81 sg_free_table(pages); 82 kfree(pages); 83 } 84 85 static void dmabuf_gem_object_free(struct kref *kref) 86 { 87 struct intel_vgpu_dmabuf_obj *obj = 88 container_of(kref, struct intel_vgpu_dmabuf_obj, kref); 89 struct intel_vgpu *vgpu = obj->vgpu; 90 struct list_head *pos; 91 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 92 93 if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) { 94 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 95 dmabuf_obj = container_of(pos, 96 struct intel_vgpu_dmabuf_obj, list); 97 if (dmabuf_obj == obj) { 98 intel_gvt_hypervisor_put_vfio_device(vgpu); 99 idr_remove(&vgpu->object_idr, 100 dmabuf_obj->dmabuf_id); 101 kfree(dmabuf_obj->info); 102 kfree(dmabuf_obj); 103 list_del(pos); 104 break; 105 } 106 } 107 } else { 108 /* Free the orphan dmabuf_objs here */ 109 kfree(obj->info); 110 kfree(obj); 111 } 112 } 113 114 115 static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj) 116 { 117 kref_get(&obj->kref); 118 } 119 120 static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj) 121 { 122 kref_put(&obj->kref, dmabuf_gem_object_free); 123 } 124 125 static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj) 126 { 127 128 struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info; 129 struct intel_vgpu_dmabuf_obj *obj = fb_info->obj; 130 struct intel_vgpu *vgpu = obj->vgpu; 131 132 if (vgpu) { 133 mutex_lock(&vgpu->dmabuf_lock); 134 gem_obj->base.dma_buf = NULL; 135 dmabuf_obj_put(obj); 136 mutex_unlock(&vgpu->dmabuf_lock); 137 } else { 138 /* vgpu is NULL, as it has been removed already */ 139 gem_obj->base.dma_buf = NULL; 140 dmabuf_obj_put(obj); 141 } 142 } 143 144 static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = { 145 .flags = I915_GEM_OBJECT_IS_PROXY, 146 .get_pages = vgpu_gem_get_pages, 147 .put_pages = vgpu_gem_put_pages, 148 .release = vgpu_gem_release, 149 }; 150 151 static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, 152 struct intel_vgpu_fb_info *info) 153 { 154 struct drm_i915_private *dev_priv = to_i915(dev); 155 struct drm_i915_gem_object *obj; 156 157 obj = i915_gem_object_alloc(dev_priv); 158 if (obj == NULL) 159 return NULL; 160 161 drm_gem_private_object_init(dev, &obj->base, 162 info->size << PAGE_SHIFT); 163 i915_gem_object_init(obj, &intel_vgpu_gem_ops); 164 165 obj->read_domains = I915_GEM_DOMAIN_GTT; 166 obj->write_domain = 0; 167 if (IS_SKYLAKE(dev_priv) 168 || IS_KABYLAKE(dev_priv) 169 || IS_BROXTON(dev_priv)) { 170 unsigned int tiling_mode = 0; 171 unsigned int stride = 0; 172 173 switch (info->drm_format_mod) { 174 case DRM_FORMAT_MOD_LINEAR: 175 tiling_mode = I915_TILING_NONE; 176 break; 177 case I915_FORMAT_MOD_X_TILED: 178 tiling_mode = I915_TILING_X; 179 stride = info->stride; 180 break; 181 case I915_FORMAT_MOD_Y_TILED: 182 case I915_FORMAT_MOD_Yf_TILED: 183 tiling_mode = I915_TILING_Y; 184 stride = info->stride; 185 break; 186 default: 187 gvt_dbg_core("invalid drm_format_mod %llx for tiling\n", 188 info->drm_format_mod); 189 } 190 obj->tiling_and_stride = tiling_mode | stride; 191 } else { 192 obj->tiling_and_stride = info->drm_format_mod ? 193 I915_TILING_X : 0; 194 } 195 196 return obj; 197 } 198 199 static bool validate_hotspot(struct intel_vgpu_cursor_plane_format *c) 200 { 201 if (c && c->x_hot <= c->width && c->y_hot <= c->height) 202 return true; 203 else 204 return false; 205 } 206 207 static int vgpu_get_plane_info(struct drm_device *dev, 208 struct intel_vgpu *vgpu, 209 struct intel_vgpu_fb_info *info, 210 int plane_id) 211 { 212 struct drm_i915_private *dev_priv = to_i915(dev); 213 struct intel_vgpu_primary_plane_format p; 214 struct intel_vgpu_cursor_plane_format c; 215 int ret; 216 217 if (plane_id == DRM_PLANE_TYPE_PRIMARY) { 218 ret = intel_vgpu_decode_primary_plane(vgpu, &p); 219 if (ret) 220 return ret; 221 info->start = p.base; 222 info->start_gpa = p.base_gpa; 223 info->width = p.width; 224 info->height = p.height; 225 info->stride = p.stride; 226 info->drm_format = p.drm_format; 227 228 switch (p.tiled) { 229 case PLANE_CTL_TILED_LINEAR: 230 info->drm_format_mod = DRM_FORMAT_MOD_LINEAR; 231 break; 232 case PLANE_CTL_TILED_X: 233 info->drm_format_mod = I915_FORMAT_MOD_X_TILED; 234 break; 235 case PLANE_CTL_TILED_Y: 236 info->drm_format_mod = I915_FORMAT_MOD_Y_TILED; 237 break; 238 case PLANE_CTL_TILED_YF: 239 info->drm_format_mod = I915_FORMAT_MOD_Yf_TILED; 240 break; 241 default: 242 gvt_vgpu_err("invalid tiling mode: %x\n", p.tiled); 243 } 244 245 info->size = (((p.stride * p.height * p.bpp) / 8) + 246 (PAGE_SIZE - 1)) >> PAGE_SHIFT; 247 } else if (plane_id == DRM_PLANE_TYPE_CURSOR) { 248 ret = intel_vgpu_decode_cursor_plane(vgpu, &c); 249 if (ret) 250 return ret; 251 info->start = c.base; 252 info->start_gpa = c.base_gpa; 253 info->width = c.width; 254 info->height = c.height; 255 info->stride = c.width * (c.bpp / 8); 256 info->drm_format = c.drm_format; 257 info->drm_format_mod = 0; 258 info->x_pos = c.x_pos; 259 info->y_pos = c.y_pos; 260 261 if (validate_hotspot(&c)) { 262 info->x_hot = c.x_hot; 263 info->y_hot = c.y_hot; 264 } else { 265 info->x_hot = UINT_MAX; 266 info->y_hot = UINT_MAX; 267 } 268 269 info->size = (((info->stride * c.height * c.bpp) / 8) 270 + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 271 } else { 272 gvt_vgpu_err("invalid plane id:%d\n", plane_id); 273 return -EINVAL; 274 } 275 276 if (info->size == 0) { 277 gvt_vgpu_err("fb size is zero\n"); 278 return -EINVAL; 279 } 280 281 if (info->start & (PAGE_SIZE - 1)) { 282 gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start); 283 return -EFAULT; 284 } 285 if (((info->start >> PAGE_SHIFT) + info->size) > 286 ggtt_total_entries(&dev_priv->ggtt)) { 287 gvt_vgpu_err("Invalid GTT offset or size\n"); 288 return -EFAULT; 289 } 290 291 if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) { 292 gvt_vgpu_err("invalid gma addr\n"); 293 return -EFAULT; 294 } 295 296 return 0; 297 } 298 299 static struct intel_vgpu_dmabuf_obj * 300 pick_dmabuf_by_info(struct intel_vgpu *vgpu, 301 struct intel_vgpu_fb_info *latest_info) 302 { 303 struct list_head *pos; 304 struct intel_vgpu_fb_info *fb_info; 305 struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; 306 struct intel_vgpu_dmabuf_obj *ret = NULL; 307 308 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 309 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 310 list); 311 if ((dmabuf_obj == NULL) || 312 (dmabuf_obj->info == NULL)) 313 continue; 314 315 fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info; 316 if ((fb_info->start == latest_info->start) && 317 (fb_info->start_gpa == latest_info->start_gpa) && 318 (fb_info->size == latest_info->size) && 319 (fb_info->drm_format_mod == latest_info->drm_format_mod) && 320 (fb_info->drm_format == latest_info->drm_format) && 321 (fb_info->width == latest_info->width) && 322 (fb_info->height == latest_info->height)) { 323 ret = dmabuf_obj; 324 break; 325 } 326 } 327 328 return ret; 329 } 330 331 static struct intel_vgpu_dmabuf_obj * 332 pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id) 333 { 334 struct list_head *pos; 335 struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; 336 struct intel_vgpu_dmabuf_obj *ret = NULL; 337 338 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 339 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 340 list); 341 if (!dmabuf_obj) 342 continue; 343 344 if (dmabuf_obj->dmabuf_id == id) { 345 ret = dmabuf_obj; 346 break; 347 } 348 } 349 350 return ret; 351 } 352 353 static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf, 354 struct intel_vgpu_fb_info *fb_info) 355 { 356 gvt_dmabuf->drm_format = fb_info->drm_format; 357 gvt_dmabuf->drm_format_mod = fb_info->drm_format_mod; 358 gvt_dmabuf->width = fb_info->width; 359 gvt_dmabuf->height = fb_info->height; 360 gvt_dmabuf->stride = fb_info->stride; 361 gvt_dmabuf->size = fb_info->size; 362 gvt_dmabuf->x_pos = fb_info->x_pos; 363 gvt_dmabuf->y_pos = fb_info->y_pos; 364 gvt_dmabuf->x_hot = fb_info->x_hot; 365 gvt_dmabuf->y_hot = fb_info->y_hot; 366 } 367 368 int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args) 369 { 370 struct drm_device *dev = &vgpu->gvt->dev_priv->drm; 371 struct vfio_device_gfx_plane_info *gfx_plane_info = args; 372 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 373 struct intel_vgpu_fb_info fb_info; 374 int ret = 0; 375 376 if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF | 377 VFIO_GFX_PLANE_TYPE_PROBE)) 378 return ret; 379 else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) || 380 (!gfx_plane_info->flags)) 381 return -EINVAL; 382 383 ret = vgpu_get_plane_info(dev, vgpu, &fb_info, 384 gfx_plane_info->drm_plane_type); 385 if (ret != 0) 386 goto out; 387 388 mutex_lock(&vgpu->dmabuf_lock); 389 /* If exists, pick up the exposed dmabuf_obj */ 390 dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info); 391 if (dmabuf_obj) { 392 update_fb_info(gfx_plane_info, &fb_info); 393 gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id; 394 395 /* This buffer may be released between query_plane ioctl and 396 * get_dmabuf ioctl. Add the refcount to make sure it won't 397 * be released between the two ioctls. 398 */ 399 if (!dmabuf_obj->initref) { 400 dmabuf_obj->initref = true; 401 dmabuf_obj_get(dmabuf_obj); 402 } 403 ret = 0; 404 gvt_dbg_dpy("vgpu%d: re-use dmabuf_obj ref %d, id %d\n", 405 vgpu->id, kref_read(&dmabuf_obj->kref), 406 gfx_plane_info->dmabuf_id); 407 mutex_unlock(&vgpu->dmabuf_lock); 408 goto out; 409 } 410 411 mutex_unlock(&vgpu->dmabuf_lock); 412 413 /* Need to allocate a new one*/ 414 dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL); 415 if (unlikely(!dmabuf_obj)) { 416 gvt_vgpu_err("alloc dmabuf_obj failed\n"); 417 ret = -ENOMEM; 418 goto out; 419 } 420 421 dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info), 422 GFP_KERNEL); 423 if (unlikely(!dmabuf_obj->info)) { 424 gvt_vgpu_err("allocate intel vgpu fb info failed\n"); 425 ret = -ENOMEM; 426 goto out_free_dmabuf; 427 } 428 memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info)); 429 430 ((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj; 431 432 dmabuf_obj->vgpu = vgpu; 433 434 ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT); 435 if (ret < 0) 436 goto out_free_info; 437 gfx_plane_info->dmabuf_id = ret; 438 dmabuf_obj->dmabuf_id = ret; 439 440 dmabuf_obj->initref = true; 441 442 kref_init(&dmabuf_obj->kref); 443 444 mutex_lock(&vgpu->dmabuf_lock); 445 if (intel_gvt_hypervisor_get_vfio_device(vgpu)) { 446 gvt_vgpu_err("get vfio device failed\n"); 447 mutex_unlock(&vgpu->dmabuf_lock); 448 goto out_free_info; 449 } 450 mutex_unlock(&vgpu->dmabuf_lock); 451 452 update_fb_info(gfx_plane_info, &fb_info); 453 454 INIT_LIST_HEAD(&dmabuf_obj->list); 455 mutex_lock(&vgpu->dmabuf_lock); 456 list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head); 457 mutex_unlock(&vgpu->dmabuf_lock); 458 459 gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id, 460 __func__, kref_read(&dmabuf_obj->kref), ret); 461 462 return 0; 463 464 out_free_info: 465 kfree(dmabuf_obj->info); 466 out_free_dmabuf: 467 kfree(dmabuf_obj); 468 out: 469 /* ENODEV means plane isn't ready, which might be a normal case. */ 470 return (ret == -ENODEV) ? 0 : ret; 471 } 472 473 /* To associate an exposed dmabuf with the dmabuf_obj */ 474 int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id) 475 { 476 struct drm_device *dev = &vgpu->gvt->dev_priv->drm; 477 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 478 struct drm_i915_gem_object *obj; 479 struct dma_buf *dmabuf; 480 int dmabuf_fd; 481 int ret = 0; 482 483 mutex_lock(&vgpu->dmabuf_lock); 484 485 dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id); 486 if (dmabuf_obj == NULL) { 487 gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id); 488 ret = -EINVAL; 489 goto out; 490 } 491 492 obj = vgpu_create_gem(dev, dmabuf_obj->info); 493 if (obj == NULL) { 494 gvt_vgpu_err("create gvt gem obj failed\n"); 495 ret = -ENOMEM; 496 goto out; 497 } 498 499 obj->gvt_info = dmabuf_obj->info; 500 501 dmabuf = i915_gem_prime_export(dev, &obj->base, DRM_CLOEXEC | DRM_RDWR); 502 if (IS_ERR(dmabuf)) { 503 gvt_vgpu_err("export dma-buf failed\n"); 504 ret = PTR_ERR(dmabuf); 505 goto out_free_gem; 506 } 507 508 i915_gem_object_put(obj); 509 510 ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR); 511 if (ret < 0) { 512 gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret); 513 goto out_free_dmabuf; 514 } 515 dmabuf_fd = ret; 516 517 dmabuf_obj_get(dmabuf_obj); 518 519 if (dmabuf_obj->initref) { 520 dmabuf_obj->initref = false; 521 dmabuf_obj_put(dmabuf_obj); 522 } 523 524 mutex_unlock(&vgpu->dmabuf_lock); 525 526 gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n" 527 " file count: %ld, GEM ref: %d\n", 528 vgpu->id, dmabuf_obj->dmabuf_id, 529 kref_read(&dmabuf_obj->kref), 530 dmabuf_fd, 531 file_count(dmabuf->file), 532 kref_read(&obj->base.refcount)); 533 534 return dmabuf_fd; 535 536 out_free_dmabuf: 537 dma_buf_put(dmabuf); 538 out_free_gem: 539 i915_gem_object_put(obj); 540 out: 541 mutex_unlock(&vgpu->dmabuf_lock); 542 return ret; 543 } 544 545 void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu) 546 { 547 struct list_head *pos, *n; 548 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 549 550 mutex_lock(&vgpu->dmabuf_lock); 551 list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) { 552 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 553 list); 554 dmabuf_obj->vgpu = NULL; 555 556 idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); 557 intel_gvt_hypervisor_put_vfio_device(vgpu); 558 list_del(pos); 559 560 /* dmabuf_obj might be freed in dmabuf_obj_put */ 561 if (dmabuf_obj->initref) { 562 dmabuf_obj->initref = false; 563 dmabuf_obj_put(dmabuf_obj); 564 } 565 566 } 567 mutex_unlock(&vgpu->dmabuf_lock); 568 } 569