1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7 #include <linux/file.h>
8 #include <linux/sync_file.h>
9 #include <linux/uaccess.h>
10
11 #include <drm/drm_drv.h>
12 #include <drm/drm_file.h>
13 #include <drm/drm_syncobj.h>
14
15 #include "msm_drv.h"
16 #include "msm_gpu.h"
17 #include "msm_gem.h"
18 #include "msm_gpu_trace.h"
19
20 /* For userspace errors, use DRM_UT_DRIVER.. so that userspace can enable
21 * error msgs for debugging, but we don't spam dmesg by default
22 */
23 #define SUBMIT_ERROR(submit, fmt, ...) \
24 DRM_DEV_DEBUG_DRIVER((submit)->dev->dev, fmt, ##__VA_ARGS__)
25
26 /*
27 * Cmdstream submission:
28 */
29
submit_create(struct drm_device * dev,struct msm_gpu * gpu,struct msm_gpu_submitqueue * queue,uint32_t nr_bos,uint32_t nr_cmds)30 static struct msm_gem_submit *submit_create(struct drm_device *dev,
31 struct msm_gpu *gpu,
32 struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
33 uint32_t nr_cmds)
34 {
35 static atomic_t ident = ATOMIC_INIT(0);
36 struct msm_gem_submit *submit;
37 uint64_t sz;
38 int ret;
39
40 sz = struct_size(submit, bos, nr_bos) +
41 ((u64)nr_cmds * sizeof(submit->cmd[0]));
42
43 if (sz > SIZE_MAX)
44 return ERR_PTR(-ENOMEM);
45
46 submit = kzalloc(sz, GFP_KERNEL);
47 if (!submit)
48 return ERR_PTR(-ENOMEM);
49
50 submit->hw_fence = msm_fence_alloc();
51 if (IS_ERR(submit->hw_fence)) {
52 ret = PTR_ERR(submit->hw_fence);
53 kfree(submit);
54 return ERR_PTR(ret);
55 }
56
57 ret = drm_sched_job_init(&submit->base, queue->entity, queue);
58 if (ret) {
59 kfree(submit->hw_fence);
60 kfree(submit);
61 return ERR_PTR(ret);
62 }
63
64 kref_init(&submit->ref);
65 submit->dev = dev;
66 submit->aspace = queue->ctx->aspace;
67 submit->gpu = gpu;
68 submit->cmd = (void *)&submit->bos[nr_bos];
69 submit->queue = queue;
70 submit->pid = get_pid(task_pid(current));
71 submit->ring = gpu->rb[queue->ring_nr];
72 submit->fault_dumped = false;
73
74 /* Get a unique identifier for the submission for logging purposes */
75 submit->ident = atomic_inc_return(&ident) - 1;
76
77 INIT_LIST_HEAD(&submit->node);
78
79 return submit;
80 }
81
__msm_gem_submit_destroy(struct kref * kref)82 void __msm_gem_submit_destroy(struct kref *kref)
83 {
84 struct msm_gem_submit *submit =
85 container_of(kref, struct msm_gem_submit, ref);
86 unsigned i;
87
88 /*
89 * In error paths, we could unref the submit without calling
90 * drm_sched_entity_push_job(), so msm_job_free() will never
91 * get called. Since drm_sched_job_cleanup() will NULL out
92 * s_fence, we can use that to detect this case.
93 */
94 if (submit->base.s_fence)
95 drm_sched_job_cleanup(&submit->base);
96
97 if (submit->fence_id) {
98 spin_lock(&submit->queue->idr_lock);
99 idr_remove(&submit->queue->fence_idr, submit->fence_id);
100 spin_unlock(&submit->queue->idr_lock);
101 }
102
103 dma_fence_put(submit->user_fence);
104
105 /*
106 * If the submit is freed before msm_job_run(), then hw_fence is
107 * just some pre-allocated memory, not a reference counted fence.
108 * Once the job runs and the hw_fence is initialized, it will
109 * have a refcount of at least one, since the submit holds a ref
110 * to the hw_fence.
111 */
112 if (kref_read(&submit->hw_fence->refcount) == 0) {
113 kfree(submit->hw_fence);
114 } else {
115 dma_fence_put(submit->hw_fence);
116 }
117
118 put_pid(submit->pid);
119 msm_submitqueue_put(submit->queue);
120
121 for (i = 0; i < submit->nr_cmds; i++)
122 kfree(submit->cmd[i].relocs);
123
124 kfree(submit);
125 }
126
submit_lookup_objects(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)127 static int submit_lookup_objects(struct msm_gem_submit *submit,
128 struct drm_msm_gem_submit *args, struct drm_file *file)
129 {
130 unsigned i;
131 int ret = 0;
132
133 for (i = 0; i < args->nr_bos; i++) {
134 struct drm_msm_gem_submit_bo submit_bo;
135 void __user *userptr =
136 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
137
138 /* make sure we don't have garbage flags, in case we hit
139 * error path before flags is initialized:
140 */
141 submit->bos[i].flags = 0;
142
143 if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
144 ret = -EFAULT;
145 i = 0;
146 goto out;
147 }
148
149 /* at least one of READ and/or WRITE flags should be set: */
150 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
151
152 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
153 !(submit_bo.flags & MANDATORY_FLAGS)) {
154 SUBMIT_ERROR(submit, "invalid flags: %x\n", submit_bo.flags);
155 ret = -EINVAL;
156 i = 0;
157 goto out;
158 }
159
160 submit->bos[i].handle = submit_bo.handle;
161 submit->bos[i].flags = submit_bo.flags;
162 /* in validate_objects() we figure out if this is true: */
163 submit->bos[i].iova = submit_bo.presumed;
164 }
165
166 spin_lock(&file->table_lock);
167
168 for (i = 0; i < args->nr_bos; i++) {
169 struct drm_gem_object *obj;
170
171 /* normally use drm_gem_object_lookup(), but for bulk lookup
172 * all under single table_lock just hit object_idr directly:
173 */
174 obj = idr_find(&file->object_idr, submit->bos[i].handle);
175 if (!obj) {
176 SUBMIT_ERROR(submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i);
177 ret = -EINVAL;
178 goto out_unlock;
179 }
180
181 drm_gem_object_get(obj);
182
183 submit->bos[i].obj = obj;
184 }
185
186 out_unlock:
187 spin_unlock(&file->table_lock);
188
189 out:
190 submit->nr_bos = i;
191
192 return ret;
193 }
194
submit_lookup_cmds(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)195 static int submit_lookup_cmds(struct msm_gem_submit *submit,
196 struct drm_msm_gem_submit *args, struct drm_file *file)
197 {
198 unsigned i;
199 size_t sz;
200 int ret = 0;
201
202 for (i = 0; i < args->nr_cmds; i++) {
203 struct drm_msm_gem_submit_cmd submit_cmd;
204 void __user *userptr =
205 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
206
207 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
208 if (ret) {
209 ret = -EFAULT;
210 goto out;
211 }
212
213 /* validate input from userspace: */
214 switch (submit_cmd.type) {
215 case MSM_SUBMIT_CMD_BUF:
216 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
217 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
218 break;
219 default:
220 SUBMIT_ERROR(submit, "invalid type: %08x\n", submit_cmd.type);
221 return -EINVAL;
222 }
223
224 if (submit_cmd.size % 4) {
225 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer size: %u\n",
226 submit_cmd.size);
227 ret = -EINVAL;
228 goto out;
229 }
230
231 submit->cmd[i].type = submit_cmd.type;
232 submit->cmd[i].size = submit_cmd.size / 4;
233 submit->cmd[i].offset = submit_cmd.submit_offset / 4;
234 submit->cmd[i].idx = submit_cmd.submit_idx;
235 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
236
237 userptr = u64_to_user_ptr(submit_cmd.relocs);
238
239 sz = array_size(submit_cmd.nr_relocs,
240 sizeof(struct drm_msm_gem_submit_reloc));
241 /* check for overflow: */
242 if (sz == SIZE_MAX) {
243 ret = -ENOMEM;
244 goto out;
245 }
246 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);
247 if (!submit->cmd[i].relocs) {
248 ret = -ENOMEM;
249 goto out;
250 }
251 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
252 if (ret) {
253 ret = -EFAULT;
254 goto out;
255 }
256 }
257
258 out:
259 return ret;
260 }
261
262 /* Unwind bo state, according to cleanup_flags. In the success case, only
263 * the lock is dropped at the end of the submit (and active/pin ref is dropped
264 * later when the submit is retired).
265 */
submit_cleanup_bo(struct msm_gem_submit * submit,int i,unsigned cleanup_flags)266 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
267 unsigned cleanup_flags)
268 {
269 struct drm_gem_object *obj = submit->bos[i].obj;
270 unsigned flags = submit->bos[i].flags & cleanup_flags;
271
272 /*
273 * Clear flags bit before dropping lock, so that the msm_job_run()
274 * path isn't racing with submit_cleanup() (ie. the read/modify/
275 * write is protected by the obj lock in all paths)
276 */
277 submit->bos[i].flags &= ~cleanup_flags;
278
279 if (flags & BO_PINNED)
280 msm_gem_unpin_locked(obj);
281
282 if (flags & BO_LOCKED)
283 dma_resv_unlock(obj->resv);
284 }
285
submit_unlock_unpin_bo(struct msm_gem_submit * submit,int i)286 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
287 {
288 unsigned cleanup_flags = BO_PINNED | BO_LOCKED;
289 submit_cleanup_bo(submit, i, cleanup_flags);
290
291 if (!(submit->bos[i].flags & BO_VALID))
292 submit->bos[i].iova = 0;
293 }
294
295 /* This is where we make sure all the bo's are reserved and pin'd: */
submit_lock_objects(struct msm_gem_submit * submit)296 static int submit_lock_objects(struct msm_gem_submit *submit)
297 {
298 int contended, slow_locked = -1, i, ret = 0;
299
300 retry:
301 for (i = 0; i < submit->nr_bos; i++) {
302 struct drm_gem_object *obj = submit->bos[i].obj;
303
304 if (slow_locked == i)
305 slow_locked = -1;
306
307 contended = i;
308
309 if (!(submit->bos[i].flags & BO_LOCKED)) {
310 ret = dma_resv_lock_interruptible(obj->resv,
311 &submit->ticket);
312 if (ret)
313 goto fail;
314 submit->bos[i].flags |= BO_LOCKED;
315 }
316 }
317
318 ww_acquire_done(&submit->ticket);
319
320 return 0;
321
322 fail:
323 if (ret == -EALREADY) {
324 SUBMIT_ERROR(submit, "handle %u at index %u already on submit list\n",
325 submit->bos[i].handle, i);
326 ret = -EINVAL;
327 }
328
329 for (; i >= 0; i--)
330 submit_unlock_unpin_bo(submit, i);
331
332 if (slow_locked > 0)
333 submit_unlock_unpin_bo(submit, slow_locked);
334
335 if (ret == -EDEADLK) {
336 struct drm_gem_object *obj = submit->bos[contended].obj;
337 /* we lost out in a seqno race, lock and retry.. */
338 ret = dma_resv_lock_slow_interruptible(obj->resv,
339 &submit->ticket);
340 if (!ret) {
341 submit->bos[contended].flags |= BO_LOCKED;
342 slow_locked = contended;
343 goto retry;
344 }
345
346 /* Not expecting -EALREADY here, if the bo was already
347 * locked, we should have gotten -EALREADY already from
348 * the dma_resv_lock_interruptable() call.
349 */
350 WARN_ON_ONCE(ret == -EALREADY);
351 }
352
353 return ret;
354 }
355
submit_fence_sync(struct msm_gem_submit * submit,bool no_implicit)356 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
357 {
358 int i, ret = 0;
359
360 for (i = 0; i < submit->nr_bos; i++) {
361 struct drm_gem_object *obj = submit->bos[i].obj;
362 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
363
364 /* NOTE: _reserve_shared() must happen before
365 * _add_shared_fence(), which makes this a slightly
366 * strange place to call it. OTOH this is a
367 * convenient can-fail point to hook it in.
368 */
369 ret = dma_resv_reserve_fences(obj->resv, 1);
370 if (ret)
371 return ret;
372
373 /* If userspace has determined that explicit fencing is
374 * used, it can disable implicit sync on the entire
375 * submit:
376 */
377 if (no_implicit)
378 continue;
379
380 /* Otherwise userspace can ask for implicit sync to be
381 * disabled on specific buffers. This is useful for internal
382 * usermode driver managed buffers, suballocation, etc.
383 */
384 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)
385 continue;
386
387 ret = drm_sched_job_add_implicit_dependencies(&submit->base,
388 obj,
389 write);
390 if (ret)
391 break;
392 }
393
394 return ret;
395 }
396
submit_pin_objects(struct msm_gem_submit * submit)397 static int submit_pin_objects(struct msm_gem_submit *submit)
398 {
399 struct msm_drm_private *priv = submit->dev->dev_private;
400 int i, ret = 0;
401
402 submit->valid = true;
403
404 for (i = 0; i < submit->nr_bos; i++) {
405 struct drm_gem_object *obj = submit->bos[i].obj;
406 struct msm_gem_vma *vma;
407
408 /* if locking succeeded, pin bo: */
409 vma = msm_gem_get_vma_locked(obj, submit->aspace);
410 if (IS_ERR(vma)) {
411 ret = PTR_ERR(vma);
412 break;
413 }
414
415 ret = msm_gem_pin_vma_locked(obj, vma);
416 if (ret)
417 break;
418
419 if (vma->iova == submit->bos[i].iova) {
420 submit->bos[i].flags |= BO_VALID;
421 } else {
422 submit->bos[i].iova = vma->iova;
423 /* iova changed, so address in cmdstream is not valid: */
424 submit->bos[i].flags &= ~BO_VALID;
425 submit->valid = false;
426 }
427 }
428
429 /*
430 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
431 * the LRU lock for each individual bo, while (b) avoiding holding the
432 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
433 * get_pages() which could trigger reclaim.. and if we held the LRU lock
434 * could trigger deadlock with the shrinker).
435 */
436 mutex_lock(&priv->lru.lock);
437 for (i = 0; i < submit->nr_bos; i++) {
438 msm_gem_pin_obj_locked(submit->bos[i].obj);
439 submit->bos[i].flags |= BO_PINNED;
440 }
441 mutex_unlock(&priv->lru.lock);
442
443 return ret;
444 }
445
submit_attach_object_fences(struct msm_gem_submit * submit)446 static void submit_attach_object_fences(struct msm_gem_submit *submit)
447 {
448 int i;
449
450 for (i = 0; i < submit->nr_bos; i++) {
451 struct drm_gem_object *obj = submit->bos[i].obj;
452
453 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
454 dma_resv_add_fence(obj->resv, submit->user_fence,
455 DMA_RESV_USAGE_WRITE);
456 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
457 dma_resv_add_fence(obj->resv, submit->user_fence,
458 DMA_RESV_USAGE_READ);
459 }
460 }
461
submit_bo(struct msm_gem_submit * submit,uint32_t idx,struct drm_gem_object ** obj,uint64_t * iova,bool * valid)462 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
463 struct drm_gem_object **obj, uint64_t *iova, bool *valid)
464 {
465 if (idx >= submit->nr_bos) {
466 SUBMIT_ERROR(submit, "invalid buffer index: %u (out of %u)\n",
467 idx, submit->nr_bos);
468 return -EINVAL;
469 }
470
471 if (obj)
472 *obj = submit->bos[idx].obj;
473 if (iova)
474 *iova = submit->bos[idx].iova;
475 if (valid)
476 *valid = !!(submit->bos[idx].flags & BO_VALID);
477
478 return 0;
479 }
480
481 /* process the reloc's and patch up the cmdstream as needed: */
submit_reloc(struct msm_gem_submit * submit,struct drm_gem_object * obj,uint32_t offset,uint32_t nr_relocs,struct drm_msm_gem_submit_reloc * relocs)482 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,
483 uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
484 {
485 uint32_t i, last_offset = 0;
486 uint32_t *ptr;
487 int ret = 0;
488
489 if (!nr_relocs)
490 return 0;
491
492 if (offset % 4) {
493 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer: %u\n", offset);
494 return -EINVAL;
495 }
496
497 /* For now, just map the entire thing. Eventually we probably
498 * to do it page-by-page, w/ kmap() if not vmap()d..
499 */
500 ptr = msm_gem_get_vaddr_locked(obj);
501
502 if (IS_ERR(ptr)) {
503 ret = PTR_ERR(ptr);
504 DBG("failed to map: %d", ret);
505 return ret;
506 }
507
508 for (i = 0; i < nr_relocs; i++) {
509 struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
510 uint32_t off;
511 uint64_t iova;
512 bool valid;
513
514 if (submit_reloc.submit_offset % 4) {
515 SUBMIT_ERROR(submit, "non-aligned reloc offset: %u\n",
516 submit_reloc.submit_offset);
517 ret = -EINVAL;
518 goto out;
519 }
520
521 /* offset in dwords: */
522 off = submit_reloc.submit_offset / 4;
523
524 if ((off >= (obj->size / 4)) ||
525 (off < last_offset)) {
526 SUBMIT_ERROR(submit, "invalid offset %u at reloc %u\n", off, i);
527 ret = -EINVAL;
528 goto out;
529 }
530
531 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
532 if (ret)
533 goto out;
534
535 if (valid)
536 continue;
537
538 iova += submit_reloc.reloc_offset;
539
540 if (submit_reloc.shift < 0)
541 iova >>= -submit_reloc.shift;
542 else
543 iova <<= submit_reloc.shift;
544
545 ptr[off] = iova | submit_reloc.or;
546
547 last_offset = off;
548 }
549
550 out:
551 msm_gem_put_vaddr_locked(obj);
552
553 return ret;
554 }
555
556 /* Cleanup submit at end of ioctl. In the error case, this also drops
557 * references, unpins, and drops active refcnt. In the non-error case,
558 * this is done when the submit is retired.
559 */
submit_cleanup(struct msm_gem_submit * submit,bool error)560 static void submit_cleanup(struct msm_gem_submit *submit, bool error)
561 {
562 unsigned cleanup_flags = BO_LOCKED;
563 unsigned i;
564
565 if (error)
566 cleanup_flags |= BO_PINNED;
567
568 for (i = 0; i < submit->nr_bos; i++) {
569 struct drm_gem_object *obj = submit->bos[i].obj;
570 submit_cleanup_bo(submit, i, cleanup_flags);
571 if (error)
572 drm_gem_object_put(obj);
573 }
574 }
575
msm_submit_retire(struct msm_gem_submit * submit)576 void msm_submit_retire(struct msm_gem_submit *submit)
577 {
578 int i;
579
580 for (i = 0; i < submit->nr_bos; i++) {
581 struct drm_gem_object *obj = submit->bos[i].obj;
582
583 drm_gem_object_put(obj);
584 }
585 }
586
587 struct msm_submit_post_dep {
588 struct drm_syncobj *syncobj;
589 uint64_t point;
590 struct dma_fence_chain *chain;
591 };
592
msm_parse_deps(struct msm_gem_submit * submit,struct drm_file * file,uint64_t in_syncobjs_addr,uint32_t nr_in_syncobjs,size_t syncobj_stride)593 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
594 struct drm_file *file,
595 uint64_t in_syncobjs_addr,
596 uint32_t nr_in_syncobjs,
597 size_t syncobj_stride)
598 {
599 struct drm_syncobj **syncobjs = NULL;
600 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
601 int ret = 0;
602 uint32_t i, j;
603
604 syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
605 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
606 if (!syncobjs)
607 return ERR_PTR(-ENOMEM);
608
609 for (i = 0; i < nr_in_syncobjs; ++i) {
610 uint64_t address = in_syncobjs_addr + i * syncobj_stride;
611
612 if (copy_from_user(&syncobj_desc,
613 u64_to_user_ptr(address),
614 min(syncobj_stride, sizeof(syncobj_desc)))) {
615 ret = -EFAULT;
616 break;
617 }
618
619 if (syncobj_desc.point &&
620 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {
621 ret = -EOPNOTSUPP;
622 break;
623 }
624
625 if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
626 ret = -EINVAL;
627 break;
628 }
629
630 ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,
631 syncobj_desc.handle, syncobj_desc.point);
632 if (ret)
633 break;
634
635 if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
636 syncobjs[i] =
637 drm_syncobj_find(file, syncobj_desc.handle);
638 if (!syncobjs[i]) {
639 ret = -EINVAL;
640 break;
641 }
642 }
643 }
644
645 if (ret) {
646 for (j = 0; j <= i; ++j) {
647 if (syncobjs[j])
648 drm_syncobj_put(syncobjs[j]);
649 }
650 kfree(syncobjs);
651 return ERR_PTR(ret);
652 }
653 return syncobjs;
654 }
655
msm_reset_syncobjs(struct drm_syncobj ** syncobjs,uint32_t nr_syncobjs)656 static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
657 uint32_t nr_syncobjs)
658 {
659 uint32_t i;
660
661 for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
662 if (syncobjs[i])
663 drm_syncobj_replace_fence(syncobjs[i], NULL);
664 }
665 }
666
msm_parse_post_deps(struct drm_device * dev,struct drm_file * file,uint64_t syncobjs_addr,uint32_t nr_syncobjs,size_t syncobj_stride)667 static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
668 struct drm_file *file,
669 uint64_t syncobjs_addr,
670 uint32_t nr_syncobjs,
671 size_t syncobj_stride)
672 {
673 struct msm_submit_post_dep *post_deps;
674 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
675 int ret = 0;
676 uint32_t i, j;
677
678 post_deps = kcalloc(nr_syncobjs, sizeof(*post_deps),
679 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
680 if (!post_deps)
681 return ERR_PTR(-ENOMEM);
682
683 for (i = 0; i < nr_syncobjs; ++i) {
684 uint64_t address = syncobjs_addr + i * syncobj_stride;
685
686 if (copy_from_user(&syncobj_desc,
687 u64_to_user_ptr(address),
688 min(syncobj_stride, sizeof(syncobj_desc)))) {
689 ret = -EFAULT;
690 break;
691 }
692
693 post_deps[i].point = syncobj_desc.point;
694
695 if (syncobj_desc.flags) {
696 ret = -EINVAL;
697 break;
698 }
699
700 if (syncobj_desc.point) {
701 if (!drm_core_check_feature(dev,
702 DRIVER_SYNCOBJ_TIMELINE)) {
703 ret = -EOPNOTSUPP;
704 break;
705 }
706
707 post_deps[i].chain = dma_fence_chain_alloc();
708 if (!post_deps[i].chain) {
709 ret = -ENOMEM;
710 break;
711 }
712 }
713
714 post_deps[i].syncobj =
715 drm_syncobj_find(file, syncobj_desc.handle);
716 if (!post_deps[i].syncobj) {
717 ret = -EINVAL;
718 break;
719 }
720 }
721
722 if (ret) {
723 for (j = 0; j <= i; ++j) {
724 dma_fence_chain_free(post_deps[j].chain);
725 if (post_deps[j].syncobj)
726 drm_syncobj_put(post_deps[j].syncobj);
727 }
728
729 kfree(post_deps);
730 return ERR_PTR(ret);
731 }
732
733 return post_deps;
734 }
735
msm_process_post_deps(struct msm_submit_post_dep * post_deps,uint32_t count,struct dma_fence * fence)736 static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
737 uint32_t count, struct dma_fence *fence)
738 {
739 uint32_t i;
740
741 for (i = 0; post_deps && i < count; ++i) {
742 if (post_deps[i].chain) {
743 drm_syncobj_add_point(post_deps[i].syncobj,
744 post_deps[i].chain,
745 fence, post_deps[i].point);
746 post_deps[i].chain = NULL;
747 } else {
748 drm_syncobj_replace_fence(post_deps[i].syncobj,
749 fence);
750 }
751 }
752 }
753
msm_ioctl_gem_submit(struct drm_device * dev,void * data,struct drm_file * file)754 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
755 struct drm_file *file)
756 {
757 struct msm_drm_private *priv = dev->dev_private;
758 struct drm_msm_gem_submit *args = data;
759 struct msm_file_private *ctx = file->driver_priv;
760 struct msm_gem_submit *submit = NULL;
761 struct msm_gpu *gpu = priv->gpu;
762 struct msm_gpu_submitqueue *queue;
763 struct msm_ringbuffer *ring;
764 struct msm_submit_post_dep *post_deps = NULL;
765 struct drm_syncobj **syncobjs_to_reset = NULL;
766 struct sync_file *sync_file = NULL;
767 int out_fence_fd = -1;
768 bool has_ww_ticket = false;
769 unsigned i;
770 int ret;
771
772 if (!gpu)
773 return -ENXIO;
774
775 if (args->pad)
776 return -EINVAL;
777
778 if (unlikely(!ctx->aspace) && !capable(CAP_SYS_RAWIO)) {
779 DRM_ERROR_RATELIMITED("IOMMU support or CAP_SYS_RAWIO required!\n");
780 return -EPERM;
781 }
782
783 /* for now, we just have 3d pipe.. eventually this would need to
784 * be more clever to dispatch to appropriate gpu module:
785 */
786 if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
787 return -EINVAL;
788
789 if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
790 return -EINVAL;
791
792 if (args->flags & MSM_SUBMIT_SUDO) {
793 if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
794 !capable(CAP_SYS_RAWIO))
795 return -EINVAL;
796 }
797
798 queue = msm_submitqueue_get(ctx, args->queueid);
799 if (!queue)
800 return -ENOENT;
801
802 ring = gpu->rb[queue->ring_nr];
803
804 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
805 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
806 if (out_fence_fd < 0) {
807 ret = out_fence_fd;
808 goto out_post_unlock;
809 }
810 }
811
812 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
813 if (IS_ERR(submit)) {
814 ret = PTR_ERR(submit);
815 goto out_post_unlock;
816 }
817
818 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
819 args->nr_bos, args->nr_cmds);
820
821 ret = mutex_lock_interruptible(&queue->lock);
822 if (ret)
823 goto out_post_unlock;
824
825 if (args->flags & MSM_SUBMIT_SUDO)
826 submit->in_rb = true;
827
828 if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
829 struct dma_fence *in_fence;
830
831 in_fence = sync_file_get_fence(args->fence_fd);
832
833 if (!in_fence) {
834 ret = -EINVAL;
835 goto out_unlock;
836 }
837
838 ret = drm_sched_job_add_dependency(&submit->base, in_fence);
839 if (ret)
840 goto out_unlock;
841 }
842
843 if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
844 syncobjs_to_reset = msm_parse_deps(submit, file,
845 args->in_syncobjs,
846 args->nr_in_syncobjs,
847 args->syncobj_stride);
848 if (IS_ERR(syncobjs_to_reset)) {
849 ret = PTR_ERR(syncobjs_to_reset);
850 goto out_unlock;
851 }
852 }
853
854 if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
855 post_deps = msm_parse_post_deps(dev, file,
856 args->out_syncobjs,
857 args->nr_out_syncobjs,
858 args->syncobj_stride);
859 if (IS_ERR(post_deps)) {
860 ret = PTR_ERR(post_deps);
861 goto out_unlock;
862 }
863 }
864
865 ret = submit_lookup_objects(submit, args, file);
866 if (ret)
867 goto out;
868
869 ret = submit_lookup_cmds(submit, args, file);
870 if (ret)
871 goto out;
872
873 /* copy_*_user while holding a ww ticket upsets lockdep */
874 ww_acquire_init(&submit->ticket, &reservation_ww_class);
875 has_ww_ticket = true;
876 ret = submit_lock_objects(submit);
877 if (ret)
878 goto out;
879
880 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
881 if (ret)
882 goto out;
883
884 ret = submit_pin_objects(submit);
885 if (ret)
886 goto out;
887
888 for (i = 0; i < args->nr_cmds; i++) {
889 struct drm_gem_object *obj;
890 uint64_t iova;
891
892 ret = submit_bo(submit, submit->cmd[i].idx,
893 &obj, &iova, NULL);
894 if (ret)
895 goto out;
896
897 if (!submit->cmd[i].size ||
898 (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) {
899 SUBMIT_ERROR(submit, "invalid cmdstream size: %u\n", submit->cmd[i].size * 4);
900 ret = -EINVAL;
901 goto out;
902 }
903
904 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
905
906 if (submit->valid)
907 continue;
908
909 if (!gpu->allow_relocs) {
910 if (submit->cmd[i].nr_relocs) {
911 SUBMIT_ERROR(submit, "relocs not allowed\n");
912 ret = -EINVAL;
913 goto out;
914 }
915
916 continue;
917 }
918
919 ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,
920 submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
921 if (ret)
922 goto out;
923 }
924
925 submit->nr_cmds = i;
926
927 idr_preload(GFP_KERNEL);
928
929 spin_lock(&queue->idr_lock);
930
931 /*
932 * If using userspace provided seqno fence, validate that the id
933 * is available before arming sched job. Since access to fence_idr
934 * is serialized on the queue lock, the slot should be still avail
935 * after the job is armed
936 */
937 if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
938 (!args->fence || idr_find(&queue->fence_idr, args->fence))) {
939 spin_unlock(&queue->idr_lock);
940 idr_preload_end();
941 ret = -EINVAL;
942 goto out;
943 }
944
945 drm_sched_job_arm(&submit->base);
946
947 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
948
949 if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
950 /*
951 * Userspace has assigned the seqno fence that it wants
952 * us to use. It is an error to pick a fence sequence
953 * number that is not available.
954 */
955 submit->fence_id = args->fence;
956 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
957 &submit->fence_id, submit->fence_id,
958 GFP_NOWAIT);
959 /*
960 * We've already validated that the fence_id slot is valid,
961 * so if idr_alloc_u32 failed, it is a kernel bug
962 */
963 WARN_ON(ret);
964 } else {
965 /*
966 * Allocate an id which can be used by WAIT_FENCE ioctl to map
967 * back to the underlying fence.
968 */
969 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
970 submit->user_fence, 1,
971 INT_MAX, GFP_NOWAIT);
972 }
973
974 spin_unlock(&queue->idr_lock);
975 idr_preload_end();
976
977 if (submit->fence_id < 0) {
978 ret = submit->fence_id;
979 submit->fence_id = 0;
980 }
981
982 if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
983 sync_file = sync_file_create(submit->user_fence);
984 if (!sync_file) {
985 ret = -ENOMEM;
986 } else {
987 fd_install(out_fence_fd, sync_file->file);
988 args->fence_fd = out_fence_fd;
989 }
990 }
991
992 submit_attach_object_fences(submit);
993
994 /* The scheduler owns a ref now: */
995 msm_gem_submit_get(submit);
996
997 msm_rd_dump_submit(priv->rd, submit, NULL);
998
999 drm_sched_entity_push_job(&submit->base);
1000
1001 args->fence = submit->fence_id;
1002 queue->last_fence = submit->fence_id;
1003
1004 msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
1005 msm_process_post_deps(post_deps, args->nr_out_syncobjs,
1006 submit->user_fence);
1007
1008
1009 out:
1010 submit_cleanup(submit, !!ret);
1011 if (has_ww_ticket)
1012 ww_acquire_fini(&submit->ticket);
1013 out_unlock:
1014 mutex_unlock(&queue->lock);
1015 out_post_unlock:
1016 if (ret && (out_fence_fd >= 0)) {
1017 put_unused_fd(out_fence_fd);
1018 if (sync_file)
1019 fput(sync_file->file);
1020 }
1021
1022 if (!IS_ERR_OR_NULL(submit)) {
1023 msm_gem_submit_put(submit);
1024 } else {
1025 /*
1026 * If the submit hasn't yet taken ownership of the queue
1027 * then we need to drop the reference ourself:
1028 */
1029 msm_submitqueue_put(queue);
1030 }
1031 if (!IS_ERR_OR_NULL(post_deps)) {
1032 for (i = 0; i < args->nr_out_syncobjs; ++i) {
1033 kfree(post_deps[i].chain);
1034 drm_syncobj_put(post_deps[i].syncobj);
1035 }
1036 kfree(post_deps);
1037 }
1038
1039 if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
1040 for (i = 0; i < args->nr_in_syncobjs; ++i) {
1041 if (syncobjs_to_reset[i])
1042 drm_syncobj_put(syncobjs_to_reset[i]);
1043 }
1044 kfree(syncobjs_to_reset);
1045 }
1046
1047 return ret;
1048 }
1049