1 /* 2 * Copyright (C) 2013 Red Hat 3 * Author: Rob Clark <robdclark@gmail.com> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published by 7 * the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/sync_file.h> 19 20 #include "msm_drv.h" 21 #include "msm_gpu.h" 22 #include "msm_gem.h" 23 #include "msm_gpu_trace.h" 24 25 /* 26 * Cmdstream submission: 27 */ 28 29 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ 30 #define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */ 31 #define BO_LOCKED 0x4000 32 #define BO_PINNED 0x2000 33 34 static struct msm_gem_submit *submit_create(struct drm_device *dev, 35 struct msm_gpu *gpu, struct msm_gpu_submitqueue *queue, 36 uint32_t nr_bos, uint32_t nr_cmds) 37 { 38 struct msm_gem_submit *submit; 39 uint64_t sz = sizeof(*submit) + ((u64)nr_bos * sizeof(submit->bos[0])) + 40 ((u64)nr_cmds * sizeof(submit->cmd[0])); 41 42 if (sz > SIZE_MAX) 43 return NULL; 44 45 submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); 46 if (!submit) 47 return NULL; 48 49 submit->dev = dev; 50 submit->gpu = gpu; 51 submit->fence = NULL; 52 submit->cmd = (void *)&submit->bos[nr_bos]; 53 submit->queue = queue; 54 submit->ring = gpu->rb[queue->prio]; 55 56 /* initially, until copy_from_user() and bo lookup succeeds: */ 57 submit->nr_bos = 0; 58 submit->nr_cmds = 0; 59 60 INIT_LIST_HEAD(&submit->node); 61 INIT_LIST_HEAD(&submit->bo_list); 62 ww_acquire_init(&submit->ticket, &reservation_ww_class); 63 64 return submit; 65 } 66 67 void msm_gem_submit_free(struct msm_gem_submit *submit) 68 { 69 dma_fence_put(submit->fence); 70 list_del(&submit->node); 71 put_pid(submit->pid); 72 msm_submitqueue_put(submit->queue); 73 74 kfree(submit); 75 } 76 77 static int submit_lookup_objects(struct msm_gem_submit *submit, 78 struct drm_msm_gem_submit *args, struct drm_file *file) 79 { 80 unsigned i; 81 int ret = 0; 82 83 for (i = 0; i < args->nr_bos; i++) { 84 struct drm_msm_gem_submit_bo submit_bo; 85 void __user *userptr = 86 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo))); 87 88 /* make sure we don't have garbage flags, in case we hit 89 * error path before flags is initialized: 90 */ 91 submit->bos[i].flags = 0; 92 93 if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) { 94 ret = -EFAULT; 95 i = 0; 96 goto out; 97 } 98 99 /* at least one of READ and/or WRITE flags should be set: */ 100 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE) 101 102 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) || 103 !(submit_bo.flags & MANDATORY_FLAGS)) { 104 DRM_ERROR("invalid flags: %x\n", submit_bo.flags); 105 ret = -EINVAL; 106 i = 0; 107 goto out; 108 } 109 110 submit->bos[i].handle = submit_bo.handle; 111 submit->bos[i].flags = submit_bo.flags; 112 /* in validate_objects() we figure out if this is true: */ 113 submit->bos[i].iova = submit_bo.presumed; 114 } 115 116 spin_lock(&file->table_lock); 117 118 for (i = 0; i < args->nr_bos; i++) { 119 struct drm_gem_object *obj; 120 struct msm_gem_object *msm_obj; 121 122 /* normally use drm_gem_object_lookup(), but for bulk lookup 123 * all under single table_lock just hit object_idr directly: 124 */ 125 obj = idr_find(&file->object_idr, submit->bos[i].handle); 126 if (!obj) { 127 DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i); 128 ret = -EINVAL; 129 goto out_unlock; 130 } 131 132 msm_obj = to_msm_bo(obj); 133 134 if (!list_empty(&msm_obj->submit_entry)) { 135 DRM_ERROR("handle %u at index %u already on submit list\n", 136 submit->bos[i].handle, i); 137 ret = -EINVAL; 138 goto out_unlock; 139 } 140 141 drm_gem_object_get(obj); 142 143 submit->bos[i].obj = msm_obj; 144 145 list_add_tail(&msm_obj->submit_entry, &submit->bo_list); 146 } 147 148 out_unlock: 149 spin_unlock(&file->table_lock); 150 151 out: 152 submit->nr_bos = i; 153 154 return ret; 155 } 156 157 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, 158 int i, bool backoff) 159 { 160 struct msm_gem_object *msm_obj = submit->bos[i].obj; 161 162 if (submit->bos[i].flags & BO_PINNED) 163 msm_gem_unpin_iova(&msm_obj->base, submit->gpu->aspace); 164 165 if (submit->bos[i].flags & BO_LOCKED) 166 ww_mutex_unlock(&msm_obj->base.resv->lock); 167 168 if (backoff && !(submit->bos[i].flags & BO_VALID)) 169 submit->bos[i].iova = 0; 170 171 submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED); 172 } 173 174 /* This is where we make sure all the bo's are reserved and pin'd: */ 175 static int submit_lock_objects(struct msm_gem_submit *submit) 176 { 177 int contended, slow_locked = -1, i, ret = 0; 178 179 retry: 180 for (i = 0; i < submit->nr_bos; i++) { 181 struct msm_gem_object *msm_obj = submit->bos[i].obj; 182 183 if (slow_locked == i) 184 slow_locked = -1; 185 186 contended = i; 187 188 if (!(submit->bos[i].flags & BO_LOCKED)) { 189 ret = ww_mutex_lock_interruptible(&msm_obj->base.resv->lock, 190 &submit->ticket); 191 if (ret) 192 goto fail; 193 submit->bos[i].flags |= BO_LOCKED; 194 } 195 } 196 197 ww_acquire_done(&submit->ticket); 198 199 return 0; 200 201 fail: 202 for (; i >= 0; i--) 203 submit_unlock_unpin_bo(submit, i, true); 204 205 if (slow_locked > 0) 206 submit_unlock_unpin_bo(submit, slow_locked, true); 207 208 if (ret == -EDEADLK) { 209 struct msm_gem_object *msm_obj = submit->bos[contended].obj; 210 /* we lost out in a seqno race, lock and retry.. */ 211 ret = ww_mutex_lock_slow_interruptible(&msm_obj->base.resv->lock, 212 &submit->ticket); 213 if (!ret) { 214 submit->bos[contended].flags |= BO_LOCKED; 215 slow_locked = contended; 216 goto retry; 217 } 218 } 219 220 return ret; 221 } 222 223 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit) 224 { 225 int i, ret = 0; 226 227 for (i = 0; i < submit->nr_bos; i++) { 228 struct msm_gem_object *msm_obj = submit->bos[i].obj; 229 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE; 230 231 if (!write) { 232 /* NOTE: _reserve_shared() must happen before 233 * _add_shared_fence(), which makes this a slightly 234 * strange place to call it. OTOH this is a 235 * convenient can-fail point to hook it in. 236 */ 237 ret = reservation_object_reserve_shared(msm_obj->base.resv, 238 1); 239 if (ret) 240 return ret; 241 } 242 243 if (no_implicit) 244 continue; 245 246 ret = msm_gem_sync_object(&msm_obj->base, submit->ring->fctx, 247 write); 248 if (ret) 249 break; 250 } 251 252 return ret; 253 } 254 255 static int submit_pin_objects(struct msm_gem_submit *submit) 256 { 257 int i, ret = 0; 258 259 submit->valid = true; 260 261 for (i = 0; i < submit->nr_bos; i++) { 262 struct msm_gem_object *msm_obj = submit->bos[i].obj; 263 uint64_t iova; 264 265 /* if locking succeeded, pin bo: */ 266 ret = msm_gem_get_and_pin_iova(&msm_obj->base, 267 submit->gpu->aspace, &iova); 268 269 if (ret) 270 break; 271 272 submit->bos[i].flags |= BO_PINNED; 273 274 if (iova == submit->bos[i].iova) { 275 submit->bos[i].flags |= BO_VALID; 276 } else { 277 submit->bos[i].iova = iova; 278 /* iova changed, so address in cmdstream is not valid: */ 279 submit->bos[i].flags &= ~BO_VALID; 280 submit->valid = false; 281 } 282 } 283 284 return ret; 285 } 286 287 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, 288 struct msm_gem_object **obj, uint64_t *iova, bool *valid) 289 { 290 if (idx >= submit->nr_bos) { 291 DRM_ERROR("invalid buffer index: %u (out of %u)\n", 292 idx, submit->nr_bos); 293 return -EINVAL; 294 } 295 296 if (obj) 297 *obj = submit->bos[idx].obj; 298 if (iova) 299 *iova = submit->bos[idx].iova; 300 if (valid) 301 *valid = !!(submit->bos[idx].flags & BO_VALID); 302 303 return 0; 304 } 305 306 /* process the reloc's and patch up the cmdstream as needed: */ 307 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj, 308 uint32_t offset, uint32_t nr_relocs, uint64_t relocs) 309 { 310 uint32_t i, last_offset = 0; 311 uint32_t *ptr; 312 int ret = 0; 313 314 if (!nr_relocs) 315 return 0; 316 317 if (offset % 4) { 318 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset); 319 return -EINVAL; 320 } 321 322 /* For now, just map the entire thing. Eventually we probably 323 * to do it page-by-page, w/ kmap() if not vmap()d.. 324 */ 325 ptr = msm_gem_get_vaddr(&obj->base); 326 327 if (IS_ERR(ptr)) { 328 ret = PTR_ERR(ptr); 329 DBG("failed to map: %d", ret); 330 return ret; 331 } 332 333 for (i = 0; i < nr_relocs; i++) { 334 struct drm_msm_gem_submit_reloc submit_reloc; 335 void __user *userptr = 336 u64_to_user_ptr(relocs + (i * sizeof(submit_reloc))); 337 uint32_t off; 338 uint64_t iova; 339 bool valid; 340 341 if (copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc))) { 342 ret = -EFAULT; 343 goto out; 344 } 345 346 if (submit_reloc.submit_offset % 4) { 347 DRM_ERROR("non-aligned reloc offset: %u\n", 348 submit_reloc.submit_offset); 349 ret = -EINVAL; 350 goto out; 351 } 352 353 /* offset in dwords: */ 354 off = submit_reloc.submit_offset / 4; 355 356 if ((off >= (obj->base.size / 4)) || 357 (off < last_offset)) { 358 DRM_ERROR("invalid offset %u at reloc %u\n", off, i); 359 ret = -EINVAL; 360 goto out; 361 } 362 363 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid); 364 if (ret) 365 goto out; 366 367 if (valid) 368 continue; 369 370 iova += submit_reloc.reloc_offset; 371 372 if (submit_reloc.shift < 0) 373 iova >>= -submit_reloc.shift; 374 else 375 iova <<= submit_reloc.shift; 376 377 ptr[off] = iova | submit_reloc.or; 378 379 last_offset = off; 380 } 381 382 out: 383 msm_gem_put_vaddr(&obj->base); 384 385 return ret; 386 } 387 388 static void submit_cleanup(struct msm_gem_submit *submit) 389 { 390 unsigned i; 391 392 for (i = 0; i < submit->nr_bos; i++) { 393 struct msm_gem_object *msm_obj = submit->bos[i].obj; 394 submit_unlock_unpin_bo(submit, i, false); 395 list_del_init(&msm_obj->submit_entry); 396 drm_gem_object_put(&msm_obj->base); 397 } 398 399 ww_acquire_fini(&submit->ticket); 400 } 401 402 int msm_ioctl_gem_submit(struct drm_device *dev, void *data, 403 struct drm_file *file) 404 { 405 static atomic_t ident = ATOMIC_INIT(0); 406 struct msm_drm_private *priv = dev->dev_private; 407 struct drm_msm_gem_submit *args = data; 408 struct msm_file_private *ctx = file->driver_priv; 409 struct msm_gem_submit *submit; 410 struct msm_gpu *gpu = priv->gpu; 411 struct sync_file *sync_file = NULL; 412 struct msm_gpu_submitqueue *queue; 413 struct msm_ringbuffer *ring; 414 int out_fence_fd = -1; 415 struct pid *pid = get_pid(task_pid(current)); 416 unsigned i; 417 int ret, submitid; 418 if (!gpu) 419 return -ENXIO; 420 421 /* for now, we just have 3d pipe.. eventually this would need to 422 * be more clever to dispatch to appropriate gpu module: 423 */ 424 if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0) 425 return -EINVAL; 426 427 if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS) 428 return -EINVAL; 429 430 if (args->flags & MSM_SUBMIT_SUDO) { 431 if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) || 432 !capable(CAP_SYS_RAWIO)) 433 return -EINVAL; 434 } 435 436 queue = msm_submitqueue_get(ctx, args->queueid); 437 if (!queue) 438 return -ENOENT; 439 440 /* Get a unique identifier for the submission for logging purposes */ 441 submitid = atomic_inc_return(&ident) - 1; 442 443 ring = gpu->rb[queue->prio]; 444 trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid, 445 args->nr_bos, args->nr_cmds); 446 447 if (args->flags & MSM_SUBMIT_FENCE_FD_IN) { 448 struct dma_fence *in_fence; 449 450 in_fence = sync_file_get_fence(args->fence_fd); 451 452 if (!in_fence) 453 return -EINVAL; 454 455 /* 456 * Wait if the fence is from a foreign context, or if the fence 457 * array contains any fence from a foreign context. 458 */ 459 ret = 0; 460 if (!dma_fence_match_context(in_fence, ring->fctx->context)) 461 ret = dma_fence_wait(in_fence, true); 462 463 dma_fence_put(in_fence); 464 if (ret) 465 return ret; 466 } 467 468 ret = mutex_lock_interruptible(&dev->struct_mutex); 469 if (ret) 470 return ret; 471 472 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) { 473 out_fence_fd = get_unused_fd_flags(O_CLOEXEC); 474 if (out_fence_fd < 0) { 475 ret = out_fence_fd; 476 goto out_unlock; 477 } 478 } 479 480 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds); 481 if (!submit) { 482 ret = -ENOMEM; 483 goto out_unlock; 484 } 485 486 submit->pid = pid; 487 submit->ident = submitid; 488 489 if (args->flags & MSM_SUBMIT_SUDO) 490 submit->in_rb = true; 491 492 ret = submit_lookup_objects(submit, args, file); 493 if (ret) 494 goto out; 495 496 ret = submit_lock_objects(submit); 497 if (ret) 498 goto out; 499 500 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT)); 501 if (ret) 502 goto out; 503 504 ret = submit_pin_objects(submit); 505 if (ret) 506 goto out; 507 508 for (i = 0; i < args->nr_cmds; i++) { 509 struct drm_msm_gem_submit_cmd submit_cmd; 510 void __user *userptr = 511 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd))); 512 struct msm_gem_object *msm_obj; 513 uint64_t iova; 514 515 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd)); 516 if (ret) { 517 ret = -EFAULT; 518 goto out; 519 } 520 521 /* validate input from userspace: */ 522 switch (submit_cmd.type) { 523 case MSM_SUBMIT_CMD_BUF: 524 case MSM_SUBMIT_CMD_IB_TARGET_BUF: 525 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: 526 break; 527 default: 528 DRM_ERROR("invalid type: %08x\n", submit_cmd.type); 529 ret = -EINVAL; 530 goto out; 531 } 532 533 ret = submit_bo(submit, submit_cmd.submit_idx, 534 &msm_obj, &iova, NULL); 535 if (ret) 536 goto out; 537 538 if (submit_cmd.size % 4) { 539 DRM_ERROR("non-aligned cmdstream buffer size: %u\n", 540 submit_cmd.size); 541 ret = -EINVAL; 542 goto out; 543 } 544 545 if (!submit_cmd.size || 546 ((submit_cmd.size + submit_cmd.submit_offset) > 547 msm_obj->base.size)) { 548 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size); 549 ret = -EINVAL; 550 goto out; 551 } 552 553 submit->cmd[i].type = submit_cmd.type; 554 submit->cmd[i].size = submit_cmd.size / 4; 555 submit->cmd[i].iova = iova + submit_cmd.submit_offset; 556 submit->cmd[i].idx = submit_cmd.submit_idx; 557 558 if (submit->valid) 559 continue; 560 561 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset, 562 submit_cmd.nr_relocs, submit_cmd.relocs); 563 if (ret) 564 goto out; 565 } 566 567 submit->nr_cmds = i; 568 569 submit->fence = msm_fence_alloc(ring->fctx); 570 if (IS_ERR(submit->fence)) { 571 ret = PTR_ERR(submit->fence); 572 submit->fence = NULL; 573 goto out; 574 } 575 576 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) { 577 sync_file = sync_file_create(submit->fence); 578 if (!sync_file) { 579 ret = -ENOMEM; 580 goto out; 581 } 582 } 583 584 msm_gpu_submit(gpu, submit, ctx); 585 586 args->fence = submit->fence->seqno; 587 588 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) { 589 fd_install(out_fence_fd, sync_file->file); 590 args->fence_fd = out_fence_fd; 591 } 592 593 out: 594 submit_cleanup(submit); 595 if (ret) 596 msm_gem_submit_free(submit); 597 out_unlock: 598 if (ret && (out_fence_fd >= 0)) 599 put_unused_fd(out_fence_fd); 600 mutex_unlock(&dev->struct_mutex); 601 return ret; 602 } 603