1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7 #include <linux/file.h>
8 #include <linux/sync_file.h>
9 #include <linux/uaccess.h>
10
11 #include <drm/drm_drv.h>
12 #include <drm/drm_file.h>
13 #include <drm/drm_syncobj.h>
14
15 #include "msm_drv.h"
16 #include "msm_gpu.h"
17 #include "msm_gem.h"
18 #include "msm_gpu_trace.h"
19
20 /* For userspace errors, use DRM_UT_DRIVER.. so that userspace can enable
21 * error msgs for debugging, but we don't spam dmesg by default
22 */
23 #define SUBMIT_ERROR(submit, fmt, ...) \
24 DRM_DEV_DEBUG_DRIVER((submit)->dev->dev, fmt, ##__VA_ARGS__)
25
26 /*
27 * Cmdstream submission:
28 */
29
submit_create(struct drm_device * dev,struct msm_gpu * gpu,struct msm_gpu_submitqueue * queue,uint32_t nr_bos,uint32_t nr_cmds)30 static struct msm_gem_submit *submit_create(struct drm_device *dev,
31 struct msm_gpu *gpu,
32 struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
33 uint32_t nr_cmds)
34 {
35 static atomic_t ident = ATOMIC_INIT(0);
36 struct msm_gem_submit *submit;
37 uint64_t sz;
38 int ret;
39
40 sz = struct_size(submit, bos, nr_bos) +
41 ((u64)nr_cmds * sizeof(submit->cmd[0]));
42
43 if (sz > SIZE_MAX)
44 return ERR_PTR(-ENOMEM);
45
46 submit = kzalloc(sz, GFP_KERNEL);
47 if (!submit)
48 return ERR_PTR(-ENOMEM);
49
50 submit->hw_fence = msm_fence_alloc();
51 if (IS_ERR(submit->hw_fence)) {
52 ret = PTR_ERR(submit->hw_fence);
53 kfree(submit);
54 return ERR_PTR(ret);
55 }
56
57 ret = drm_sched_job_init(&submit->base, queue->entity, queue);
58 if (ret) {
59 kfree(submit->hw_fence);
60 kfree(submit);
61 return ERR_PTR(ret);
62 }
63
64 kref_init(&submit->ref);
65 submit->dev = dev;
66 submit->aspace = queue->ctx->aspace;
67 submit->gpu = gpu;
68 submit->cmd = (void *)&submit->bos[nr_bos];
69 submit->queue = queue;
70 submit->pid = get_pid(task_pid(current));
71 submit->ring = gpu->rb[queue->ring_nr];
72 submit->fault_dumped = false;
73
74 /* Get a unique identifier for the submission for logging purposes */
75 submit->ident = atomic_inc_return(&ident) - 1;
76
77 INIT_LIST_HEAD(&submit->node);
78
79 return submit;
80 }
81
__msm_gem_submit_destroy(struct kref * kref)82 void __msm_gem_submit_destroy(struct kref *kref)
83 {
84 struct msm_gem_submit *submit =
85 container_of(kref, struct msm_gem_submit, ref);
86 unsigned i;
87
88 if (submit->fence_id) {
89 spin_lock(&submit->queue->idr_lock);
90 idr_remove(&submit->queue->fence_idr, submit->fence_id);
91 spin_unlock(&submit->queue->idr_lock);
92 }
93
94 dma_fence_put(submit->user_fence);
95
96 /*
97 * If the submit is freed before msm_job_run(), then hw_fence is
98 * just some pre-allocated memory, not a reference counted fence.
99 * Once the job runs and the hw_fence is initialized, it will
100 * have a refcount of at least one, since the submit holds a ref
101 * to the hw_fence.
102 */
103 if (kref_read(&submit->hw_fence->refcount) == 0) {
104 kfree(submit->hw_fence);
105 } else {
106 dma_fence_put(submit->hw_fence);
107 }
108
109 put_pid(submit->pid);
110 msm_submitqueue_put(submit->queue);
111
112 for (i = 0; i < submit->nr_cmds; i++)
113 kfree(submit->cmd[i].relocs);
114
115 kfree(submit);
116 }
117
submit_lookup_objects(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)118 static int submit_lookup_objects(struct msm_gem_submit *submit,
119 struct drm_msm_gem_submit *args, struct drm_file *file)
120 {
121 unsigned i;
122 int ret = 0;
123
124 for (i = 0; i < args->nr_bos; i++) {
125 struct drm_msm_gem_submit_bo submit_bo;
126 void __user *userptr =
127 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
128
129 /* make sure we don't have garbage flags, in case we hit
130 * error path before flags is initialized:
131 */
132 submit->bos[i].flags = 0;
133
134 if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
135 ret = -EFAULT;
136 i = 0;
137 goto out;
138 }
139
140 /* at least one of READ and/or WRITE flags should be set: */
141 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
142
143 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
144 !(submit_bo.flags & MANDATORY_FLAGS)) {
145 SUBMIT_ERROR(submit, "invalid flags: %x\n", submit_bo.flags);
146 ret = -EINVAL;
147 i = 0;
148 goto out;
149 }
150
151 submit->bos[i].handle = submit_bo.handle;
152 submit->bos[i].flags = submit_bo.flags;
153 /* in validate_objects() we figure out if this is true: */
154 submit->bos[i].iova = submit_bo.presumed;
155 }
156
157 spin_lock(&file->table_lock);
158
159 for (i = 0; i < args->nr_bos; i++) {
160 struct drm_gem_object *obj;
161
162 /* normally use drm_gem_object_lookup(), but for bulk lookup
163 * all under single table_lock just hit object_idr directly:
164 */
165 obj = idr_find(&file->object_idr, submit->bos[i].handle);
166 if (!obj) {
167 SUBMIT_ERROR(submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i);
168 ret = -EINVAL;
169 goto out_unlock;
170 }
171
172 drm_gem_object_get(obj);
173
174 submit->bos[i].obj = obj;
175 }
176
177 out_unlock:
178 spin_unlock(&file->table_lock);
179
180 out:
181 submit->nr_bos = i;
182
183 return ret;
184 }
185
submit_lookup_cmds(struct msm_gem_submit * submit,struct drm_msm_gem_submit * args,struct drm_file * file)186 static int submit_lookup_cmds(struct msm_gem_submit *submit,
187 struct drm_msm_gem_submit *args, struct drm_file *file)
188 {
189 unsigned i;
190 size_t sz;
191 int ret = 0;
192
193 for (i = 0; i < args->nr_cmds; i++) {
194 struct drm_msm_gem_submit_cmd submit_cmd;
195 void __user *userptr =
196 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
197
198 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
199 if (ret) {
200 ret = -EFAULT;
201 goto out;
202 }
203
204 /* validate input from userspace: */
205 switch (submit_cmd.type) {
206 case MSM_SUBMIT_CMD_BUF:
207 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
208 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
209 break;
210 default:
211 SUBMIT_ERROR(submit, "invalid type: %08x\n", submit_cmd.type);
212 return -EINVAL;
213 }
214
215 if (submit_cmd.size % 4) {
216 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer size: %u\n",
217 submit_cmd.size);
218 ret = -EINVAL;
219 goto out;
220 }
221
222 submit->cmd[i].type = submit_cmd.type;
223 submit->cmd[i].size = submit_cmd.size / 4;
224 submit->cmd[i].offset = submit_cmd.submit_offset / 4;
225 submit->cmd[i].idx = submit_cmd.submit_idx;
226 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
227
228 userptr = u64_to_user_ptr(submit_cmd.relocs);
229
230 sz = array_size(submit_cmd.nr_relocs,
231 sizeof(struct drm_msm_gem_submit_reloc));
232 /* check for overflow: */
233 if (sz == SIZE_MAX) {
234 ret = -ENOMEM;
235 goto out;
236 }
237 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);
238 if (!submit->cmd[i].relocs) {
239 ret = -ENOMEM;
240 goto out;
241 }
242 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
243 if (ret) {
244 ret = -EFAULT;
245 goto out;
246 }
247 }
248
249 out:
250 return ret;
251 }
252
253 /* Unwind bo state, according to cleanup_flags. In the success case, only
254 * the lock is dropped at the end of the submit (and active/pin ref is dropped
255 * later when the submit is retired).
256 */
submit_cleanup_bo(struct msm_gem_submit * submit,int i,unsigned cleanup_flags)257 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
258 unsigned cleanup_flags)
259 {
260 struct drm_gem_object *obj = submit->bos[i].obj;
261 unsigned flags = submit->bos[i].flags & cleanup_flags;
262
263 /*
264 * Clear flags bit before dropping lock, so that the msm_job_run()
265 * path isn't racing with submit_cleanup() (ie. the read/modify/
266 * write is protected by the obj lock in all paths)
267 */
268 submit->bos[i].flags &= ~cleanup_flags;
269
270 if (flags & BO_PINNED)
271 msm_gem_unpin_locked(obj);
272
273 if (flags & BO_LOCKED)
274 dma_resv_unlock(obj->resv);
275 }
276
submit_unlock_unpin_bo(struct msm_gem_submit * submit,int i)277 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
278 {
279 unsigned cleanup_flags = BO_PINNED | BO_LOCKED;
280 submit_cleanup_bo(submit, i, cleanup_flags);
281
282 if (!(submit->bos[i].flags & BO_VALID))
283 submit->bos[i].iova = 0;
284 }
285
286 /* This is where we make sure all the bo's are reserved and pin'd: */
submit_lock_objects(struct msm_gem_submit * submit)287 static int submit_lock_objects(struct msm_gem_submit *submit)
288 {
289 int contended, slow_locked = -1, i, ret = 0;
290
291 retry:
292 for (i = 0; i < submit->nr_bos; i++) {
293 struct drm_gem_object *obj = submit->bos[i].obj;
294
295 if (slow_locked == i)
296 slow_locked = -1;
297
298 contended = i;
299
300 if (!(submit->bos[i].flags & BO_LOCKED)) {
301 ret = dma_resv_lock_interruptible(obj->resv,
302 &submit->ticket);
303 if (ret)
304 goto fail;
305 submit->bos[i].flags |= BO_LOCKED;
306 }
307 }
308
309 ww_acquire_done(&submit->ticket);
310
311 return 0;
312
313 fail:
314 if (ret == -EALREADY) {
315 SUBMIT_ERROR(submit, "handle %u at index %u already on submit list\n",
316 submit->bos[i].handle, i);
317 ret = -EINVAL;
318 }
319
320 for (; i >= 0; i--)
321 submit_unlock_unpin_bo(submit, i);
322
323 if (slow_locked > 0)
324 submit_unlock_unpin_bo(submit, slow_locked);
325
326 if (ret == -EDEADLK) {
327 struct drm_gem_object *obj = submit->bos[contended].obj;
328 /* we lost out in a seqno race, lock and retry.. */
329 ret = dma_resv_lock_slow_interruptible(obj->resv,
330 &submit->ticket);
331 if (!ret) {
332 submit->bos[contended].flags |= BO_LOCKED;
333 slow_locked = contended;
334 goto retry;
335 }
336
337 /* Not expecting -EALREADY here, if the bo was already
338 * locked, we should have gotten -EALREADY already from
339 * the dma_resv_lock_interruptable() call.
340 */
341 WARN_ON_ONCE(ret == -EALREADY);
342 }
343
344 return ret;
345 }
346
submit_fence_sync(struct msm_gem_submit * submit,bool no_implicit)347 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
348 {
349 int i, ret = 0;
350
351 for (i = 0; i < submit->nr_bos; i++) {
352 struct drm_gem_object *obj = submit->bos[i].obj;
353 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
354
355 /* NOTE: _reserve_shared() must happen before
356 * _add_shared_fence(), which makes this a slightly
357 * strange place to call it. OTOH this is a
358 * convenient can-fail point to hook it in.
359 */
360 ret = dma_resv_reserve_fences(obj->resv, 1);
361 if (ret)
362 return ret;
363
364 /* If userspace has determined that explicit fencing is
365 * used, it can disable implicit sync on the entire
366 * submit:
367 */
368 if (no_implicit)
369 continue;
370
371 /* Otherwise userspace can ask for implicit sync to be
372 * disabled on specific buffers. This is useful for internal
373 * usermode driver managed buffers, suballocation, etc.
374 */
375 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)
376 continue;
377
378 ret = drm_sched_job_add_implicit_dependencies(&submit->base,
379 obj,
380 write);
381 if (ret)
382 break;
383 }
384
385 return ret;
386 }
387
submit_pin_objects(struct msm_gem_submit * submit)388 static int submit_pin_objects(struct msm_gem_submit *submit)
389 {
390 struct msm_drm_private *priv = submit->dev->dev_private;
391 int i, ret = 0;
392
393 submit->valid = true;
394
395 for (i = 0; i < submit->nr_bos; i++) {
396 struct drm_gem_object *obj = submit->bos[i].obj;
397 struct msm_gem_vma *vma;
398
399 /* if locking succeeded, pin bo: */
400 vma = msm_gem_get_vma_locked(obj, submit->aspace);
401 if (IS_ERR(vma)) {
402 ret = PTR_ERR(vma);
403 break;
404 }
405
406 ret = msm_gem_pin_vma_locked(obj, vma);
407 if (ret)
408 break;
409
410 if (vma->iova == submit->bos[i].iova) {
411 submit->bos[i].flags |= BO_VALID;
412 } else {
413 submit->bos[i].iova = vma->iova;
414 /* iova changed, so address in cmdstream is not valid: */
415 submit->bos[i].flags &= ~BO_VALID;
416 submit->valid = false;
417 }
418 }
419
420 /*
421 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
422 * the LRU lock for each individual bo, while (b) avoiding holding the
423 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
424 * get_pages() which could trigger reclaim.. and if we held the LRU lock
425 * could trigger deadlock with the shrinker).
426 */
427 mutex_lock(&priv->lru.lock);
428 for (i = 0; i < submit->nr_bos; i++) {
429 msm_gem_pin_obj_locked(submit->bos[i].obj);
430 submit->bos[i].flags |= BO_PINNED;
431 }
432 mutex_unlock(&priv->lru.lock);
433
434 return ret;
435 }
436
submit_attach_object_fences(struct msm_gem_submit * submit)437 static void submit_attach_object_fences(struct msm_gem_submit *submit)
438 {
439 int i;
440
441 for (i = 0; i < submit->nr_bos; i++) {
442 struct drm_gem_object *obj = submit->bos[i].obj;
443
444 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
445 dma_resv_add_fence(obj->resv, submit->user_fence,
446 DMA_RESV_USAGE_WRITE);
447 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
448 dma_resv_add_fence(obj->resv, submit->user_fence,
449 DMA_RESV_USAGE_READ);
450 }
451 }
452
submit_bo(struct msm_gem_submit * submit,uint32_t idx,struct drm_gem_object ** obj,uint64_t * iova,bool * valid)453 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
454 struct drm_gem_object **obj, uint64_t *iova, bool *valid)
455 {
456 if (idx >= submit->nr_bos) {
457 SUBMIT_ERROR(submit, "invalid buffer index: %u (out of %u)\n",
458 idx, submit->nr_bos);
459 return -EINVAL;
460 }
461
462 if (obj)
463 *obj = submit->bos[idx].obj;
464 if (iova)
465 *iova = submit->bos[idx].iova;
466 if (valid)
467 *valid = !!(submit->bos[idx].flags & BO_VALID);
468
469 return 0;
470 }
471
472 /* process the reloc's and patch up the cmdstream as needed: */
submit_reloc(struct msm_gem_submit * submit,struct drm_gem_object * obj,uint32_t offset,uint32_t nr_relocs,struct drm_msm_gem_submit_reloc * relocs)473 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,
474 uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
475 {
476 uint32_t i, last_offset = 0;
477 uint32_t *ptr;
478 int ret = 0;
479
480 if (!nr_relocs)
481 return 0;
482
483 if (offset % 4) {
484 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer: %u\n", offset);
485 return -EINVAL;
486 }
487
488 /* For now, just map the entire thing. Eventually we probably
489 * to do it page-by-page, w/ kmap() if not vmap()d..
490 */
491 ptr = msm_gem_get_vaddr_locked(obj);
492
493 if (IS_ERR(ptr)) {
494 ret = PTR_ERR(ptr);
495 DBG("failed to map: %d", ret);
496 return ret;
497 }
498
499 for (i = 0; i < nr_relocs; i++) {
500 struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
501 uint32_t off;
502 uint64_t iova;
503 bool valid;
504
505 if (submit_reloc.submit_offset % 4) {
506 SUBMIT_ERROR(submit, "non-aligned reloc offset: %u\n",
507 submit_reloc.submit_offset);
508 ret = -EINVAL;
509 goto out;
510 }
511
512 /* offset in dwords: */
513 off = submit_reloc.submit_offset / 4;
514
515 if ((off >= (obj->size / 4)) ||
516 (off < last_offset)) {
517 SUBMIT_ERROR(submit, "invalid offset %u at reloc %u\n", off, i);
518 ret = -EINVAL;
519 goto out;
520 }
521
522 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
523 if (ret)
524 goto out;
525
526 if (valid)
527 continue;
528
529 iova += submit_reloc.reloc_offset;
530
531 if (submit_reloc.shift < 0)
532 iova >>= -submit_reloc.shift;
533 else
534 iova <<= submit_reloc.shift;
535
536 ptr[off] = iova | submit_reloc.or;
537
538 last_offset = off;
539 }
540
541 out:
542 msm_gem_put_vaddr_locked(obj);
543
544 return ret;
545 }
546
547 /* Cleanup submit at end of ioctl. In the error case, this also drops
548 * references, unpins, and drops active refcnt. In the non-error case,
549 * this is done when the submit is retired.
550 */
submit_cleanup(struct msm_gem_submit * submit,bool error)551 static void submit_cleanup(struct msm_gem_submit *submit, bool error)
552 {
553 unsigned cleanup_flags = BO_LOCKED;
554 unsigned i;
555
556 if (error)
557 cleanup_flags |= BO_PINNED;
558
559 for (i = 0; i < submit->nr_bos; i++) {
560 struct drm_gem_object *obj = submit->bos[i].obj;
561 submit_cleanup_bo(submit, i, cleanup_flags);
562 if (error)
563 drm_gem_object_put(obj);
564 }
565 }
566
msm_submit_retire(struct msm_gem_submit * submit)567 void msm_submit_retire(struct msm_gem_submit *submit)
568 {
569 int i;
570
571 for (i = 0; i < submit->nr_bos; i++) {
572 struct drm_gem_object *obj = submit->bos[i].obj;
573
574 drm_gem_object_put(obj);
575 }
576 }
577
578 struct msm_submit_post_dep {
579 struct drm_syncobj *syncobj;
580 uint64_t point;
581 struct dma_fence_chain *chain;
582 };
583
msm_parse_deps(struct msm_gem_submit * submit,struct drm_file * file,uint64_t in_syncobjs_addr,uint32_t nr_in_syncobjs,size_t syncobj_stride)584 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
585 struct drm_file *file,
586 uint64_t in_syncobjs_addr,
587 uint32_t nr_in_syncobjs,
588 size_t syncobj_stride)
589 {
590 struct drm_syncobj **syncobjs = NULL;
591 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
592 int ret = 0;
593 uint32_t i, j;
594
595 syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
596 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
597 if (!syncobjs)
598 return ERR_PTR(-ENOMEM);
599
600 for (i = 0; i < nr_in_syncobjs; ++i) {
601 uint64_t address = in_syncobjs_addr + i * syncobj_stride;
602
603 if (copy_from_user(&syncobj_desc,
604 u64_to_user_ptr(address),
605 min(syncobj_stride, sizeof(syncobj_desc)))) {
606 ret = -EFAULT;
607 break;
608 }
609
610 if (syncobj_desc.point &&
611 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {
612 ret = -EOPNOTSUPP;
613 break;
614 }
615
616 if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
617 ret = -EINVAL;
618 break;
619 }
620
621 ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,
622 syncobj_desc.handle, syncobj_desc.point);
623 if (ret)
624 break;
625
626 if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
627 syncobjs[i] =
628 drm_syncobj_find(file, syncobj_desc.handle);
629 if (!syncobjs[i]) {
630 ret = -EINVAL;
631 break;
632 }
633 }
634 }
635
636 if (ret) {
637 for (j = 0; j <= i; ++j) {
638 if (syncobjs[j])
639 drm_syncobj_put(syncobjs[j]);
640 }
641 kfree(syncobjs);
642 return ERR_PTR(ret);
643 }
644 return syncobjs;
645 }
646
msm_reset_syncobjs(struct drm_syncobj ** syncobjs,uint32_t nr_syncobjs)647 static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
648 uint32_t nr_syncobjs)
649 {
650 uint32_t i;
651
652 for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
653 if (syncobjs[i])
654 drm_syncobj_replace_fence(syncobjs[i], NULL);
655 }
656 }
657
msm_parse_post_deps(struct drm_device * dev,struct drm_file * file,uint64_t syncobjs_addr,uint32_t nr_syncobjs,size_t syncobj_stride)658 static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
659 struct drm_file *file,
660 uint64_t syncobjs_addr,
661 uint32_t nr_syncobjs,
662 size_t syncobj_stride)
663 {
664 struct msm_submit_post_dep *post_deps;
665 struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
666 int ret = 0;
667 uint32_t i, j;
668
669 post_deps = kcalloc(nr_syncobjs, sizeof(*post_deps),
670 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
671 if (!post_deps)
672 return ERR_PTR(-ENOMEM);
673
674 for (i = 0; i < nr_syncobjs; ++i) {
675 uint64_t address = syncobjs_addr + i * syncobj_stride;
676
677 if (copy_from_user(&syncobj_desc,
678 u64_to_user_ptr(address),
679 min(syncobj_stride, sizeof(syncobj_desc)))) {
680 ret = -EFAULT;
681 break;
682 }
683
684 post_deps[i].point = syncobj_desc.point;
685
686 if (syncobj_desc.flags) {
687 ret = -EINVAL;
688 break;
689 }
690
691 if (syncobj_desc.point) {
692 if (!drm_core_check_feature(dev,
693 DRIVER_SYNCOBJ_TIMELINE)) {
694 ret = -EOPNOTSUPP;
695 break;
696 }
697
698 post_deps[i].chain = dma_fence_chain_alloc();
699 if (!post_deps[i].chain) {
700 ret = -ENOMEM;
701 break;
702 }
703 }
704
705 post_deps[i].syncobj =
706 drm_syncobj_find(file, syncobj_desc.handle);
707 if (!post_deps[i].syncobj) {
708 ret = -EINVAL;
709 break;
710 }
711 }
712
713 if (ret) {
714 for (j = 0; j <= i; ++j) {
715 dma_fence_chain_free(post_deps[j].chain);
716 if (post_deps[j].syncobj)
717 drm_syncobj_put(post_deps[j].syncobj);
718 }
719
720 kfree(post_deps);
721 return ERR_PTR(ret);
722 }
723
724 return post_deps;
725 }
726
msm_process_post_deps(struct msm_submit_post_dep * post_deps,uint32_t count,struct dma_fence * fence)727 static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
728 uint32_t count, struct dma_fence *fence)
729 {
730 uint32_t i;
731
732 for (i = 0; post_deps && i < count; ++i) {
733 if (post_deps[i].chain) {
734 drm_syncobj_add_point(post_deps[i].syncobj,
735 post_deps[i].chain,
736 fence, post_deps[i].point);
737 post_deps[i].chain = NULL;
738 } else {
739 drm_syncobj_replace_fence(post_deps[i].syncobj,
740 fence);
741 }
742 }
743 }
744
msm_ioctl_gem_submit(struct drm_device * dev,void * data,struct drm_file * file)745 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
746 struct drm_file *file)
747 {
748 struct msm_drm_private *priv = dev->dev_private;
749 struct drm_msm_gem_submit *args = data;
750 struct msm_file_private *ctx = file->driver_priv;
751 struct msm_gem_submit *submit = NULL;
752 struct msm_gpu *gpu = priv->gpu;
753 struct msm_gpu_submitqueue *queue;
754 struct msm_ringbuffer *ring;
755 struct msm_submit_post_dep *post_deps = NULL;
756 struct drm_syncobj **syncobjs_to_reset = NULL;
757 int out_fence_fd = -1;
758 bool has_ww_ticket = false;
759 unsigned i;
760 int ret;
761
762 if (!gpu)
763 return -ENXIO;
764
765 if (args->pad)
766 return -EINVAL;
767
768 if (unlikely(!ctx->aspace) && !capable(CAP_SYS_RAWIO)) {
769 DRM_ERROR_RATELIMITED("IOMMU support or CAP_SYS_RAWIO required!\n");
770 return -EPERM;
771 }
772
773 /* for now, we just have 3d pipe.. eventually this would need to
774 * be more clever to dispatch to appropriate gpu module:
775 */
776 if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
777 return -EINVAL;
778
779 if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
780 return -EINVAL;
781
782 if (args->flags & MSM_SUBMIT_SUDO) {
783 if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
784 !capable(CAP_SYS_RAWIO))
785 return -EINVAL;
786 }
787
788 queue = msm_submitqueue_get(ctx, args->queueid);
789 if (!queue)
790 return -ENOENT;
791
792 ring = gpu->rb[queue->ring_nr];
793
794 if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
795 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
796 if (out_fence_fd < 0) {
797 ret = out_fence_fd;
798 goto out_post_unlock;
799 }
800 }
801
802 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
803 if (IS_ERR(submit)) {
804 ret = PTR_ERR(submit);
805 goto out_post_unlock;
806 }
807
808 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
809 args->nr_bos, args->nr_cmds);
810
811 ret = mutex_lock_interruptible(&queue->lock);
812 if (ret)
813 goto out_post_unlock;
814
815 if (args->flags & MSM_SUBMIT_SUDO)
816 submit->in_rb = true;
817
818 if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
819 struct dma_fence *in_fence;
820
821 in_fence = sync_file_get_fence(args->fence_fd);
822
823 if (!in_fence) {
824 ret = -EINVAL;
825 goto out_unlock;
826 }
827
828 ret = drm_sched_job_add_dependency(&submit->base, in_fence);
829 if (ret)
830 goto out_unlock;
831 }
832
833 if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
834 syncobjs_to_reset = msm_parse_deps(submit, file,
835 args->in_syncobjs,
836 args->nr_in_syncobjs,
837 args->syncobj_stride);
838 if (IS_ERR(syncobjs_to_reset)) {
839 ret = PTR_ERR(syncobjs_to_reset);
840 goto out_unlock;
841 }
842 }
843
844 if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
845 post_deps = msm_parse_post_deps(dev, file,
846 args->out_syncobjs,
847 args->nr_out_syncobjs,
848 args->syncobj_stride);
849 if (IS_ERR(post_deps)) {
850 ret = PTR_ERR(post_deps);
851 goto out_unlock;
852 }
853 }
854
855 ret = submit_lookup_objects(submit, args, file);
856 if (ret)
857 goto out;
858
859 ret = submit_lookup_cmds(submit, args, file);
860 if (ret)
861 goto out;
862
863 /* copy_*_user while holding a ww ticket upsets lockdep */
864 ww_acquire_init(&submit->ticket, &reservation_ww_class);
865 has_ww_ticket = true;
866 ret = submit_lock_objects(submit);
867 if (ret)
868 goto out;
869
870 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
871 if (ret)
872 goto out;
873
874 ret = submit_pin_objects(submit);
875 if (ret)
876 goto out;
877
878 for (i = 0; i < args->nr_cmds; i++) {
879 struct drm_gem_object *obj;
880 uint64_t iova;
881
882 ret = submit_bo(submit, submit->cmd[i].idx,
883 &obj, &iova, NULL);
884 if (ret)
885 goto out;
886
887 if (!submit->cmd[i].size ||
888 (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) {
889 SUBMIT_ERROR(submit, "invalid cmdstream size: %u\n", submit->cmd[i].size * 4);
890 ret = -EINVAL;
891 goto out;
892 }
893
894 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
895
896 if (submit->valid)
897 continue;
898
899 if (!gpu->allow_relocs) {
900 if (submit->cmd[i].nr_relocs) {
901 SUBMIT_ERROR(submit, "relocs not allowed\n");
902 ret = -EINVAL;
903 goto out;
904 }
905
906 continue;
907 }
908
909 ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,
910 submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
911 if (ret)
912 goto out;
913 }
914
915 submit->nr_cmds = i;
916
917 idr_preload(GFP_KERNEL);
918
919 spin_lock(&queue->idr_lock);
920
921 /*
922 * If using userspace provided seqno fence, validate that the id
923 * is available before arming sched job. Since access to fence_idr
924 * is serialized on the queue lock, the slot should be still avail
925 * after the job is armed
926 */
927 if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
928 (!args->fence || idr_find(&queue->fence_idr, args->fence))) {
929 spin_unlock(&queue->idr_lock);
930 idr_preload_end();
931 ret = -EINVAL;
932 goto out;
933 }
934
935 drm_sched_job_arm(&submit->base);
936
937 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
938
939 if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
940 /*
941 * Userspace has assigned the seqno fence that it wants
942 * us to use. It is an error to pick a fence sequence
943 * number that is not available.
944 */
945 submit->fence_id = args->fence;
946 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
947 &submit->fence_id, submit->fence_id,
948 GFP_NOWAIT);
949 /*
950 * We've already validated that the fence_id slot is valid,
951 * so if idr_alloc_u32 failed, it is a kernel bug
952 */
953 WARN_ON(ret);
954 } else {
955 /*
956 * Allocate an id which can be used by WAIT_FENCE ioctl to map
957 * back to the underlying fence.
958 */
959 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
960 submit->user_fence, 1,
961 INT_MAX, GFP_NOWAIT);
962 }
963
964 spin_unlock(&queue->idr_lock);
965 idr_preload_end();
966
967 if (submit->fence_id < 0) {
968 ret = submit->fence_id;
969 submit->fence_id = 0;
970 }
971
972 if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
973 struct sync_file *sync_file = sync_file_create(submit->user_fence);
974 if (!sync_file) {
975 ret = -ENOMEM;
976 } else {
977 fd_install(out_fence_fd, sync_file->file);
978 args->fence_fd = out_fence_fd;
979 }
980 }
981
982 submit_attach_object_fences(submit);
983
984 /* The scheduler owns a ref now: */
985 msm_gem_submit_get(submit);
986
987 msm_rd_dump_submit(priv->rd, submit, NULL);
988
989 drm_sched_entity_push_job(&submit->base);
990
991 args->fence = submit->fence_id;
992 queue->last_fence = submit->fence_id;
993
994 msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
995 msm_process_post_deps(post_deps, args->nr_out_syncobjs,
996 submit->user_fence);
997
998
999 out:
1000 submit_cleanup(submit, !!ret);
1001 if (has_ww_ticket)
1002 ww_acquire_fini(&submit->ticket);
1003 out_unlock:
1004 mutex_unlock(&queue->lock);
1005 out_post_unlock:
1006 if (ret && (out_fence_fd >= 0))
1007 put_unused_fd(out_fence_fd);
1008
1009 if (!IS_ERR_OR_NULL(submit)) {
1010 msm_gem_submit_put(submit);
1011 } else {
1012 /*
1013 * If the submit hasn't yet taken ownership of the queue
1014 * then we need to drop the reference ourself:
1015 */
1016 msm_submitqueue_put(queue);
1017 }
1018 if (!IS_ERR_OR_NULL(post_deps)) {
1019 for (i = 0; i < args->nr_out_syncobjs; ++i) {
1020 kfree(post_deps[i].chain);
1021 drm_syncobj_put(post_deps[i].syncobj);
1022 }
1023 kfree(post_deps);
1024 }
1025
1026 if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
1027 for (i = 0; i < args->nr_in_syncobjs; ++i) {
1028 if (syncobjs_to_reset[i])
1029 drm_syncobj_put(syncobjs_to_reset[i]);
1030 }
1031 kfree(syncobjs_to_reset);
1032 }
1033
1034 return ret;
1035 }
1036