1 /* 2 * Copyright 2017 Intel Corporation. All rights reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: 24 * Zhiyuan Lv <zhiyuan.lv@intel.com> 25 * 26 * Contributors: 27 * Xiaoguang Chen 28 * Tina Zhang <tina.zhang@intel.com> 29 */ 30 31 #include <linux/dma-buf.h> 32 #include <linux/vfio.h> 33 34 #include "i915_drv.h" 35 #include "gvt.h" 36 37 #define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12)) 38 39 static int vgpu_gem_get_pages( 40 struct drm_i915_gem_object *obj) 41 { 42 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 43 struct sg_table *st; 44 struct scatterlist *sg; 45 int i, ret; 46 gen8_pte_t __iomem *gtt_entries; 47 struct intel_vgpu_fb_info *fb_info; 48 u32 page_num; 49 50 fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info; 51 if (WARN_ON(!fb_info)) 52 return -ENODEV; 53 54 st = kmalloc(sizeof(*st), GFP_KERNEL); 55 if (unlikely(!st)) 56 return -ENOMEM; 57 58 page_num = obj->base.size >> PAGE_SHIFT; 59 ret = sg_alloc_table(st, page_num, GFP_KERNEL); 60 if (ret) { 61 kfree(st); 62 return ret; 63 } 64 gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + 65 (fb_info->start >> PAGE_SHIFT); 66 for_each_sg(st->sgl, sg, page_num, i) { 67 sg->offset = 0; 68 sg->length = PAGE_SIZE; 69 sg_dma_address(sg) = 70 GEN8_DECODE_PTE(readq(>t_entries[i])); 71 sg_dma_len(sg) = PAGE_SIZE; 72 } 73 74 __i915_gem_object_set_pages(obj, st, PAGE_SIZE); 75 76 return 0; 77 } 78 79 static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj, 80 struct sg_table *pages) 81 { 82 sg_free_table(pages); 83 kfree(pages); 84 } 85 86 static void dmabuf_gem_object_free(struct kref *kref) 87 { 88 struct intel_vgpu_dmabuf_obj *obj = 89 container_of(kref, struct intel_vgpu_dmabuf_obj, kref); 90 struct intel_vgpu *vgpu = obj->vgpu; 91 struct list_head *pos; 92 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 93 94 if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) { 95 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 96 dmabuf_obj = container_of(pos, 97 struct intel_vgpu_dmabuf_obj, list); 98 if (dmabuf_obj == obj) { 99 intel_gvt_hypervisor_put_vfio_device(vgpu); 100 idr_remove(&vgpu->object_idr, 101 dmabuf_obj->dmabuf_id); 102 kfree(dmabuf_obj->info); 103 kfree(dmabuf_obj); 104 list_del(pos); 105 break; 106 } 107 } 108 } else { 109 /* Free the orphan dmabuf_objs here */ 110 kfree(obj->info); 111 kfree(obj); 112 } 113 } 114 115 116 static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj) 117 { 118 kref_get(&obj->kref); 119 } 120 121 static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj) 122 { 123 kref_put(&obj->kref, dmabuf_gem_object_free); 124 } 125 126 static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj) 127 { 128 129 struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info; 130 struct intel_vgpu_dmabuf_obj *obj = fb_info->obj; 131 struct intel_vgpu *vgpu = obj->vgpu; 132 133 if (vgpu) { 134 mutex_lock(&vgpu->dmabuf_lock); 135 gem_obj->base.dma_buf = NULL; 136 dmabuf_obj_put(obj); 137 mutex_unlock(&vgpu->dmabuf_lock); 138 } else { 139 /* vgpu is NULL, as it has been removed already */ 140 gem_obj->base.dma_buf = NULL; 141 dmabuf_obj_put(obj); 142 } 143 } 144 145 static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = { 146 .flags = I915_GEM_OBJECT_IS_PROXY, 147 .get_pages = vgpu_gem_get_pages, 148 .put_pages = vgpu_gem_put_pages, 149 .release = vgpu_gem_release, 150 }; 151 152 static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, 153 struct intel_vgpu_fb_info *info) 154 { 155 static struct lock_class_key lock_class; 156 struct drm_i915_private *dev_priv = to_i915(dev); 157 struct drm_i915_gem_object *obj; 158 159 obj = i915_gem_object_alloc(); 160 if (obj == NULL) 161 return NULL; 162 163 drm_gem_private_object_init(dev, &obj->base, 164 roundup(info->size, PAGE_SIZE)); 165 i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class); 166 167 obj->read_domains = I915_GEM_DOMAIN_GTT; 168 obj->write_domain = 0; 169 if (INTEL_GEN(dev_priv) >= 9) { 170 unsigned int tiling_mode = 0; 171 unsigned int stride = 0; 172 173 switch (info->drm_format_mod) { 174 case DRM_FORMAT_MOD_LINEAR: 175 tiling_mode = I915_TILING_NONE; 176 break; 177 case I915_FORMAT_MOD_X_TILED: 178 tiling_mode = I915_TILING_X; 179 stride = info->stride; 180 break; 181 case I915_FORMAT_MOD_Y_TILED: 182 case I915_FORMAT_MOD_Yf_TILED: 183 tiling_mode = I915_TILING_Y; 184 stride = info->stride; 185 break; 186 default: 187 gvt_dbg_core("invalid drm_format_mod %llx for tiling\n", 188 info->drm_format_mod); 189 } 190 obj->tiling_and_stride = tiling_mode | stride; 191 } else { 192 obj->tiling_and_stride = info->drm_format_mod ? 193 I915_TILING_X : 0; 194 } 195 196 return obj; 197 } 198 199 static bool validate_hotspot(struct intel_vgpu_cursor_plane_format *c) 200 { 201 if (c && c->x_hot <= c->width && c->y_hot <= c->height) 202 return true; 203 else 204 return false; 205 } 206 207 static int vgpu_get_plane_info(struct drm_device *dev, 208 struct intel_vgpu *vgpu, 209 struct intel_vgpu_fb_info *info, 210 int plane_id) 211 { 212 struct intel_vgpu_primary_plane_format p; 213 struct intel_vgpu_cursor_plane_format c; 214 int ret, tile_height = 1; 215 216 memset(info, 0, sizeof(*info)); 217 218 if (plane_id == DRM_PLANE_TYPE_PRIMARY) { 219 ret = intel_vgpu_decode_primary_plane(vgpu, &p); 220 if (ret) 221 return ret; 222 info->start = p.base; 223 info->start_gpa = p.base_gpa; 224 info->width = p.width; 225 info->height = p.height; 226 info->stride = p.stride; 227 info->drm_format = p.drm_format; 228 229 switch (p.tiled) { 230 case PLANE_CTL_TILED_LINEAR: 231 info->drm_format_mod = DRM_FORMAT_MOD_LINEAR; 232 break; 233 case PLANE_CTL_TILED_X: 234 info->drm_format_mod = I915_FORMAT_MOD_X_TILED; 235 tile_height = 8; 236 break; 237 case PLANE_CTL_TILED_Y: 238 info->drm_format_mod = I915_FORMAT_MOD_Y_TILED; 239 tile_height = 32; 240 break; 241 case PLANE_CTL_TILED_YF: 242 info->drm_format_mod = I915_FORMAT_MOD_Yf_TILED; 243 tile_height = 32; 244 break; 245 default: 246 gvt_vgpu_err("invalid tiling mode: %x\n", p.tiled); 247 } 248 } else if (plane_id == DRM_PLANE_TYPE_CURSOR) { 249 ret = intel_vgpu_decode_cursor_plane(vgpu, &c); 250 if (ret) 251 return ret; 252 info->start = c.base; 253 info->start_gpa = c.base_gpa; 254 info->width = c.width; 255 info->height = c.height; 256 info->stride = c.width * (c.bpp / 8); 257 info->drm_format = c.drm_format; 258 info->drm_format_mod = 0; 259 info->x_pos = c.x_pos; 260 info->y_pos = c.y_pos; 261 262 if (validate_hotspot(&c)) { 263 info->x_hot = c.x_hot; 264 info->y_hot = c.y_hot; 265 } else { 266 info->x_hot = UINT_MAX; 267 info->y_hot = UINT_MAX; 268 } 269 } else { 270 gvt_vgpu_err("invalid plane id:%d\n", plane_id); 271 return -EINVAL; 272 } 273 274 info->size = info->stride * roundup(info->height, tile_height); 275 if (info->size == 0) { 276 gvt_vgpu_err("fb size is zero\n"); 277 return -EINVAL; 278 } 279 280 if (info->start & (PAGE_SIZE - 1)) { 281 gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start); 282 return -EFAULT; 283 } 284 285 if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) { 286 gvt_vgpu_err("invalid gma addr\n"); 287 return -EFAULT; 288 } 289 290 return 0; 291 } 292 293 static struct intel_vgpu_dmabuf_obj * 294 pick_dmabuf_by_info(struct intel_vgpu *vgpu, 295 struct intel_vgpu_fb_info *latest_info) 296 { 297 struct list_head *pos; 298 struct intel_vgpu_fb_info *fb_info; 299 struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; 300 struct intel_vgpu_dmabuf_obj *ret = NULL; 301 302 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 303 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 304 list); 305 if ((dmabuf_obj == NULL) || 306 (dmabuf_obj->info == NULL)) 307 continue; 308 309 fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info; 310 if ((fb_info->start == latest_info->start) && 311 (fb_info->start_gpa == latest_info->start_gpa) && 312 (fb_info->size == latest_info->size) && 313 (fb_info->drm_format_mod == latest_info->drm_format_mod) && 314 (fb_info->drm_format == latest_info->drm_format) && 315 (fb_info->width == latest_info->width) && 316 (fb_info->height == latest_info->height)) { 317 ret = dmabuf_obj; 318 break; 319 } 320 } 321 322 return ret; 323 } 324 325 static struct intel_vgpu_dmabuf_obj * 326 pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id) 327 { 328 struct list_head *pos; 329 struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; 330 struct intel_vgpu_dmabuf_obj *ret = NULL; 331 332 list_for_each(pos, &vgpu->dmabuf_obj_list_head) { 333 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 334 list); 335 if (!dmabuf_obj) 336 continue; 337 338 if (dmabuf_obj->dmabuf_id == id) { 339 ret = dmabuf_obj; 340 break; 341 } 342 } 343 344 return ret; 345 } 346 347 static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf, 348 struct intel_vgpu_fb_info *fb_info) 349 { 350 gvt_dmabuf->drm_format = fb_info->drm_format; 351 gvt_dmabuf->drm_format_mod = fb_info->drm_format_mod; 352 gvt_dmabuf->width = fb_info->width; 353 gvt_dmabuf->height = fb_info->height; 354 gvt_dmabuf->stride = fb_info->stride; 355 gvt_dmabuf->size = fb_info->size; 356 gvt_dmabuf->x_pos = fb_info->x_pos; 357 gvt_dmabuf->y_pos = fb_info->y_pos; 358 gvt_dmabuf->x_hot = fb_info->x_hot; 359 gvt_dmabuf->y_hot = fb_info->y_hot; 360 } 361 362 int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args) 363 { 364 struct drm_device *dev = &vgpu->gvt->dev_priv->drm; 365 struct vfio_device_gfx_plane_info *gfx_plane_info = args; 366 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 367 struct intel_vgpu_fb_info fb_info; 368 int ret = 0; 369 370 if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF | 371 VFIO_GFX_PLANE_TYPE_PROBE)) 372 return ret; 373 else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) || 374 (!gfx_plane_info->flags)) 375 return -EINVAL; 376 377 ret = vgpu_get_plane_info(dev, vgpu, &fb_info, 378 gfx_plane_info->drm_plane_type); 379 if (ret != 0) 380 goto out; 381 382 mutex_lock(&vgpu->dmabuf_lock); 383 /* If exists, pick up the exposed dmabuf_obj */ 384 dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info); 385 if (dmabuf_obj) { 386 update_fb_info(gfx_plane_info, &fb_info); 387 gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id; 388 389 /* This buffer may be released between query_plane ioctl and 390 * get_dmabuf ioctl. Add the refcount to make sure it won't 391 * be released between the two ioctls. 392 */ 393 if (!dmabuf_obj->initref) { 394 dmabuf_obj->initref = true; 395 dmabuf_obj_get(dmabuf_obj); 396 } 397 ret = 0; 398 gvt_dbg_dpy("vgpu%d: re-use dmabuf_obj ref %d, id %d\n", 399 vgpu->id, kref_read(&dmabuf_obj->kref), 400 gfx_plane_info->dmabuf_id); 401 mutex_unlock(&vgpu->dmabuf_lock); 402 goto out; 403 } 404 405 mutex_unlock(&vgpu->dmabuf_lock); 406 407 /* Need to allocate a new one*/ 408 dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL); 409 if (unlikely(!dmabuf_obj)) { 410 gvt_vgpu_err("alloc dmabuf_obj failed\n"); 411 ret = -ENOMEM; 412 goto out; 413 } 414 415 dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info), 416 GFP_KERNEL); 417 if (unlikely(!dmabuf_obj->info)) { 418 gvt_vgpu_err("allocate intel vgpu fb info failed\n"); 419 ret = -ENOMEM; 420 goto out_free_dmabuf; 421 } 422 memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info)); 423 424 ((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj; 425 426 dmabuf_obj->vgpu = vgpu; 427 428 ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT); 429 if (ret < 0) 430 goto out_free_info; 431 gfx_plane_info->dmabuf_id = ret; 432 dmabuf_obj->dmabuf_id = ret; 433 434 dmabuf_obj->initref = true; 435 436 kref_init(&dmabuf_obj->kref); 437 438 mutex_lock(&vgpu->dmabuf_lock); 439 if (intel_gvt_hypervisor_get_vfio_device(vgpu)) { 440 gvt_vgpu_err("get vfio device failed\n"); 441 mutex_unlock(&vgpu->dmabuf_lock); 442 goto out_free_info; 443 } 444 mutex_unlock(&vgpu->dmabuf_lock); 445 446 update_fb_info(gfx_plane_info, &fb_info); 447 448 INIT_LIST_HEAD(&dmabuf_obj->list); 449 mutex_lock(&vgpu->dmabuf_lock); 450 list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head); 451 mutex_unlock(&vgpu->dmabuf_lock); 452 453 gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id, 454 __func__, kref_read(&dmabuf_obj->kref), ret); 455 456 return 0; 457 458 out_free_info: 459 kfree(dmabuf_obj->info); 460 out_free_dmabuf: 461 kfree(dmabuf_obj); 462 out: 463 /* ENODEV means plane isn't ready, which might be a normal case. */ 464 return (ret == -ENODEV) ? 0 : ret; 465 } 466 467 /* To associate an exposed dmabuf with the dmabuf_obj */ 468 int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id) 469 { 470 struct drm_device *dev = &vgpu->gvt->dev_priv->drm; 471 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 472 struct drm_i915_gem_object *obj; 473 struct dma_buf *dmabuf; 474 int dmabuf_fd; 475 int ret = 0; 476 477 mutex_lock(&vgpu->dmabuf_lock); 478 479 dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id); 480 if (dmabuf_obj == NULL) { 481 gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id); 482 ret = -EINVAL; 483 goto out; 484 } 485 486 obj = vgpu_create_gem(dev, dmabuf_obj->info); 487 if (obj == NULL) { 488 gvt_vgpu_err("create gvt gem obj failed\n"); 489 ret = -ENOMEM; 490 goto out; 491 } 492 493 obj->gvt_info = dmabuf_obj->info; 494 495 dmabuf = i915_gem_prime_export(&obj->base, DRM_CLOEXEC | DRM_RDWR); 496 if (IS_ERR(dmabuf)) { 497 gvt_vgpu_err("export dma-buf failed\n"); 498 ret = PTR_ERR(dmabuf); 499 goto out_free_gem; 500 } 501 502 ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR); 503 if (ret < 0) { 504 gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret); 505 goto out_free_dmabuf; 506 } 507 dmabuf_fd = ret; 508 509 dmabuf_obj_get(dmabuf_obj); 510 511 if (dmabuf_obj->initref) { 512 dmabuf_obj->initref = false; 513 dmabuf_obj_put(dmabuf_obj); 514 } 515 516 mutex_unlock(&vgpu->dmabuf_lock); 517 518 gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n" 519 " file count: %ld, GEM ref: %d\n", 520 vgpu->id, dmabuf_obj->dmabuf_id, 521 kref_read(&dmabuf_obj->kref), 522 dmabuf_fd, 523 file_count(dmabuf->file), 524 kref_read(&obj->base.refcount)); 525 526 i915_gem_object_put(obj); 527 528 return dmabuf_fd; 529 530 out_free_dmabuf: 531 dma_buf_put(dmabuf); 532 out_free_gem: 533 i915_gem_object_put(obj); 534 out: 535 mutex_unlock(&vgpu->dmabuf_lock); 536 return ret; 537 } 538 539 void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu) 540 { 541 struct list_head *pos, *n; 542 struct intel_vgpu_dmabuf_obj *dmabuf_obj; 543 544 mutex_lock(&vgpu->dmabuf_lock); 545 list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) { 546 dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, 547 list); 548 dmabuf_obj->vgpu = NULL; 549 550 idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); 551 intel_gvt_hypervisor_put_vfio_device(vgpu); 552 list_del(pos); 553 554 /* dmabuf_obj might be freed in dmabuf_obj_put */ 555 if (dmabuf_obj->initref) { 556 dmabuf_obj->initref = false; 557 dmabuf_obj_put(dmabuf_obj); 558 } 559 560 } 561 mutex_unlock(&vgpu->dmabuf_lock); 562 } 563