1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include <linux/file.h>
8 #include <linux/sync_file.h>
9 #include <linux/uaccess.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_file.h>
13 #include <drm/drm_syncobj.h>
14 
15 #include "msm_drv.h"
16 #include "msm_gpu.h"
17 #include "msm_gem.h"
18 #include "msm_gpu_trace.h"
19 
20 /*
21  * Cmdstream submission:
22  */
23 
24 static struct msm_gem_submit *submit_create(struct drm_device *dev,
25 		struct msm_gpu *gpu,
26 		struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
27 		uint32_t nr_cmds)
28 {
29 	static atomic_t ident = ATOMIC_INIT(0);
30 	struct msm_gem_submit *submit;
31 	uint64_t sz;
32 	int ret;
33 
34 	sz = struct_size(submit, bos, nr_bos) +
35 			((u64)nr_cmds * sizeof(submit->cmd[0]));
36 
37 	if (sz > SIZE_MAX)
38 		return ERR_PTR(-ENOMEM);
39 
40 	submit = kzalloc(sz, GFP_KERNEL);
41 	if (!submit)
42 		return ERR_PTR(-ENOMEM);
43 
44 	submit->hw_fence = msm_fence_alloc();
45 	if (IS_ERR(submit->hw_fence)) {
46 		ret = PTR_ERR(submit->hw_fence);
47 		kfree(submit);
48 		return ERR_PTR(ret);
49 	}
50 
51 	ret = drm_sched_job_init(&submit->base, queue->entity, queue);
52 	if (ret) {
53 		kfree(submit->hw_fence);
54 		kfree(submit);
55 		return ERR_PTR(ret);
56 	}
57 
58 	kref_init(&submit->ref);
59 	submit->dev = dev;
60 	submit->aspace = queue->ctx->aspace;
61 	submit->gpu = gpu;
62 	submit->cmd = (void *)&submit->bos[nr_bos];
63 	submit->queue = queue;
64 	submit->pid = get_pid(task_pid(current));
65 	submit->ring = gpu->rb[queue->ring_nr];
66 	submit->fault_dumped = false;
67 
68 	/* Get a unique identifier for the submission for logging purposes */
69 	submit->ident = atomic_inc_return(&ident) - 1;
70 
71 	INIT_LIST_HEAD(&submit->node);
72 
73 	return submit;
74 }
75 
76 void __msm_gem_submit_destroy(struct kref *kref)
77 {
78 	struct msm_gem_submit *submit =
79 			container_of(kref, struct msm_gem_submit, ref);
80 	unsigned i;
81 
82 	if (submit->fence_id) {
83 		spin_lock(&submit->queue->idr_lock);
84 		idr_remove(&submit->queue->fence_idr, submit->fence_id);
85 		spin_unlock(&submit->queue->idr_lock);
86 	}
87 
88 	dma_fence_put(submit->user_fence);
89 
90 	/*
91 	 * If the submit is freed before msm_job_run(), then hw_fence is
92 	 * just some pre-allocated memory, not a reference counted fence.
93 	 * Once the job runs and the hw_fence is initialized, it will
94 	 * have a refcount of at least one, since the submit holds a ref
95 	 * to the hw_fence.
96 	 */
97 	if (kref_read(&submit->hw_fence->refcount) == 0) {
98 		kfree(submit->hw_fence);
99 	} else {
100 		dma_fence_put(submit->hw_fence);
101 	}
102 
103 	put_pid(submit->pid);
104 	msm_submitqueue_put(submit->queue);
105 
106 	for (i = 0; i < submit->nr_cmds; i++)
107 		kfree(submit->cmd[i].relocs);
108 
109 	kfree(submit);
110 }
111 
112 static int submit_lookup_objects(struct msm_gem_submit *submit,
113 		struct drm_msm_gem_submit *args, struct drm_file *file)
114 {
115 	unsigned i;
116 	int ret = 0;
117 
118 	for (i = 0; i < args->nr_bos; i++) {
119 		struct drm_msm_gem_submit_bo submit_bo;
120 		void __user *userptr =
121 			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
122 
123 		/* make sure we don't have garbage flags, in case we hit
124 		 * error path before flags is initialized:
125 		 */
126 		submit->bos[i].flags = 0;
127 
128 		if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
129 			ret = -EFAULT;
130 			i = 0;
131 			goto out;
132 		}
133 
134 /* at least one of READ and/or WRITE flags should be set: */
135 #define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
136 
137 		if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
138 			!(submit_bo.flags & MANDATORY_FLAGS)) {
139 			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
140 			ret = -EINVAL;
141 			i = 0;
142 			goto out;
143 		}
144 
145 		submit->bos[i].handle = submit_bo.handle;
146 		submit->bos[i].flags = submit_bo.flags;
147 		/* in validate_objects() we figure out if this is true: */
148 		submit->bos[i].iova  = submit_bo.presumed;
149 	}
150 
151 	spin_lock(&file->table_lock);
152 
153 	for (i = 0; i < args->nr_bos; i++) {
154 		struct drm_gem_object *obj;
155 
156 		/* normally use drm_gem_object_lookup(), but for bulk lookup
157 		 * all under single table_lock just hit object_idr directly:
158 		 */
159 		obj = idr_find(&file->object_idr, submit->bos[i].handle);
160 		if (!obj) {
161 			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);
162 			ret = -EINVAL;
163 			goto out_unlock;
164 		}
165 
166 		drm_gem_object_get(obj);
167 
168 		submit->bos[i].obj = obj;
169 	}
170 
171 out_unlock:
172 	spin_unlock(&file->table_lock);
173 
174 out:
175 	submit->nr_bos = i;
176 
177 	return ret;
178 }
179 
180 static int submit_lookup_cmds(struct msm_gem_submit *submit,
181 		struct drm_msm_gem_submit *args, struct drm_file *file)
182 {
183 	unsigned i;
184 	size_t sz;
185 	int ret = 0;
186 
187 	for (i = 0; i < args->nr_cmds; i++) {
188 		struct drm_msm_gem_submit_cmd submit_cmd;
189 		void __user *userptr =
190 			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
191 
192 		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
193 		if (ret) {
194 			ret = -EFAULT;
195 			goto out;
196 		}
197 
198 		/* validate input from userspace: */
199 		switch (submit_cmd.type) {
200 		case MSM_SUBMIT_CMD_BUF:
201 		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
202 		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
203 			break;
204 		default:
205 			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
206 			return -EINVAL;
207 		}
208 
209 		if (submit_cmd.size % 4) {
210 			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
211 					submit_cmd.size);
212 			ret = -EINVAL;
213 			goto out;
214 		}
215 
216 		submit->cmd[i].type = submit_cmd.type;
217 		submit->cmd[i].size = submit_cmd.size / 4;
218 		submit->cmd[i].offset = submit_cmd.submit_offset / 4;
219 		submit->cmd[i].idx  = submit_cmd.submit_idx;
220 		submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
221 
222 		userptr = u64_to_user_ptr(submit_cmd.relocs);
223 
224 		sz = array_size(submit_cmd.nr_relocs,
225 				sizeof(struct drm_msm_gem_submit_reloc));
226 		/* check for overflow: */
227 		if (sz == SIZE_MAX) {
228 			ret = -ENOMEM;
229 			goto out;
230 		}
231 		submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL);
232 		if (!submit->cmd[i].relocs) {
233 			ret = -ENOMEM;
234 			goto out;
235 		}
236 		ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
237 		if (ret) {
238 			ret = -EFAULT;
239 			goto out;
240 		}
241 	}
242 
243 out:
244 	return ret;
245 }
246 
247 /* Unwind bo state, according to cleanup_flags.  In the success case, only
248  * the lock is dropped at the end of the submit (and active/pin ref is dropped
249  * later when the submit is retired).
250  */
251 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
252 		unsigned cleanup_flags)
253 {
254 	struct drm_gem_object *obj = submit->bos[i].obj;
255 	unsigned flags = submit->bos[i].flags & cleanup_flags;
256 
257 	/*
258 	 * Clear flags bit before dropping lock, so that the msm_job_run()
259 	 * path isn't racing with submit_cleanup() (ie. the read/modify/
260 	 * write is protected by the obj lock in all paths)
261 	 */
262 	submit->bos[i].flags &= ~cleanup_flags;
263 
264 	if (flags & BO_PINNED)
265 		msm_gem_unpin_locked(obj);
266 
267 	if (flags & BO_LOCKED)
268 		dma_resv_unlock(obj->resv);
269 }
270 
271 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
272 {
273 	unsigned cleanup_flags = BO_PINNED | BO_LOCKED;
274 	submit_cleanup_bo(submit, i, cleanup_flags);
275 
276 	if (!(submit->bos[i].flags & BO_VALID))
277 		submit->bos[i].iova = 0;
278 }
279 
280 /* This is where we make sure all the bo's are reserved and pin'd: */
281 static int submit_lock_objects(struct msm_gem_submit *submit)
282 {
283 	int contended, slow_locked = -1, i, ret = 0;
284 
285 retry:
286 	for (i = 0; i < submit->nr_bos; i++) {
287 		struct drm_gem_object *obj = submit->bos[i].obj;
288 
289 		if (slow_locked == i)
290 			slow_locked = -1;
291 
292 		contended = i;
293 
294 		if (!(submit->bos[i].flags & BO_LOCKED)) {
295 			ret = dma_resv_lock_interruptible(obj->resv,
296 							  &submit->ticket);
297 			if (ret)
298 				goto fail;
299 			submit->bos[i].flags |= BO_LOCKED;
300 		}
301 	}
302 
303 	ww_acquire_done(&submit->ticket);
304 
305 	return 0;
306 
307 fail:
308 	if (ret == -EALREADY) {
309 		DRM_ERROR("handle %u at index %u already on submit list\n",
310 				submit->bos[i].handle, i);
311 		ret = -EINVAL;
312 	}
313 
314 	for (; i >= 0; i--)
315 		submit_unlock_unpin_bo(submit, i);
316 
317 	if (slow_locked > 0)
318 		submit_unlock_unpin_bo(submit, slow_locked);
319 
320 	if (ret == -EDEADLK) {
321 		struct drm_gem_object *obj = submit->bos[contended].obj;
322 		/* we lost out in a seqno race, lock and retry.. */
323 		ret = dma_resv_lock_slow_interruptible(obj->resv,
324 						       &submit->ticket);
325 		if (!ret) {
326 			submit->bos[contended].flags |= BO_LOCKED;
327 			slow_locked = contended;
328 			goto retry;
329 		}
330 
331 		/* Not expecting -EALREADY here, if the bo was already
332 		 * locked, we should have gotten -EALREADY already from
333 		 * the dma_resv_lock_interruptable() call.
334 		 */
335 		WARN_ON_ONCE(ret == -EALREADY);
336 	}
337 
338 	return ret;
339 }
340 
341 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
342 {
343 	int i, ret = 0;
344 
345 	for (i = 0; i < submit->nr_bos; i++) {
346 		struct drm_gem_object *obj = submit->bos[i].obj;
347 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
348 
349 		/* NOTE: _reserve_shared() must happen before
350 		 * _add_shared_fence(), which makes this a slightly
351 		 * strange place to call it.  OTOH this is a
352 		 * convenient can-fail point to hook it in.
353 		 */
354 		ret = dma_resv_reserve_fences(obj->resv, 1);
355 		if (ret)
356 			return ret;
357 
358 		/* If userspace has determined that explicit fencing is
359 		 * used, it can disable implicit sync on the entire
360 		 * submit:
361 		 */
362 		if (no_implicit)
363 			continue;
364 
365 		/* Otherwise userspace can ask for implicit sync to be
366 		 * disabled on specific buffers.  This is useful for internal
367 		 * usermode driver managed buffers, suballocation, etc.
368 		 */
369 		if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)
370 			continue;
371 
372 		ret = drm_sched_job_add_implicit_dependencies(&submit->base,
373 							      obj,
374 							      write);
375 		if (ret)
376 			break;
377 	}
378 
379 	return ret;
380 }
381 
382 static int submit_pin_objects(struct msm_gem_submit *submit)
383 {
384 	struct msm_drm_private *priv = submit->dev->dev_private;
385 	int i, ret = 0;
386 
387 	submit->valid = true;
388 
389 	for (i = 0; i < submit->nr_bos; i++) {
390 		struct drm_gem_object *obj = submit->bos[i].obj;
391 		struct msm_gem_vma *vma;
392 
393 		/* if locking succeeded, pin bo: */
394 		vma = msm_gem_get_vma_locked(obj, submit->aspace);
395 		if (IS_ERR(vma)) {
396 			ret = PTR_ERR(vma);
397 			break;
398 		}
399 
400 		ret = msm_gem_pin_vma_locked(obj, vma);
401 		if (ret)
402 			break;
403 
404 		if (vma->iova == submit->bos[i].iova) {
405 			submit->bos[i].flags |= BO_VALID;
406 		} else {
407 			submit->bos[i].iova = vma->iova;
408 			/* iova changed, so address in cmdstream is not valid: */
409 			submit->bos[i].flags &= ~BO_VALID;
410 			submit->valid = false;
411 		}
412 	}
413 
414 	/*
415 	 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
416 	 * the LRU lock for each individual bo, while (b) avoiding holding the
417 	 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
418 	 * get_pages() which could trigger reclaim.. and if we held the LRU lock
419 	 * could trigger deadlock with the shrinker).
420 	 */
421 	mutex_lock(&priv->lru.lock);
422 	for (i = 0; i < submit->nr_bos; i++) {
423 		msm_gem_pin_obj_locked(submit->bos[i].obj);
424 		submit->bos[i].flags |= BO_PINNED;
425 	}
426 	mutex_unlock(&priv->lru.lock);
427 
428 	return ret;
429 }
430 
431 static void submit_attach_object_fences(struct msm_gem_submit *submit)
432 {
433 	int i;
434 
435 	for (i = 0; i < submit->nr_bos; i++) {
436 		struct drm_gem_object *obj = submit->bos[i].obj;
437 
438 		if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
439 			dma_resv_add_fence(obj->resv, submit->user_fence,
440 					   DMA_RESV_USAGE_WRITE);
441 		else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
442 			dma_resv_add_fence(obj->resv, submit->user_fence,
443 					   DMA_RESV_USAGE_READ);
444 	}
445 }
446 
447 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
448 		struct drm_gem_object **obj, uint64_t *iova, bool *valid)
449 {
450 	if (idx >= submit->nr_bos) {
451 		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
452 				idx, submit->nr_bos);
453 		return -EINVAL;
454 	}
455 
456 	if (obj)
457 		*obj = submit->bos[idx].obj;
458 	if (iova)
459 		*iova = submit->bos[idx].iova;
460 	if (valid)
461 		*valid = !!(submit->bos[idx].flags & BO_VALID);
462 
463 	return 0;
464 }
465 
466 /* process the reloc's and patch up the cmdstream as needed: */
467 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,
468 		uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
469 {
470 	uint32_t i, last_offset = 0;
471 	uint32_t *ptr;
472 	int ret = 0;
473 
474 	if (!nr_relocs)
475 		return 0;
476 
477 	if (offset % 4) {
478 		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
479 		return -EINVAL;
480 	}
481 
482 	/* For now, just map the entire thing.  Eventually we probably
483 	 * to do it page-by-page, w/ kmap() if not vmap()d..
484 	 */
485 	ptr = msm_gem_get_vaddr_locked(obj);
486 
487 	if (IS_ERR(ptr)) {
488 		ret = PTR_ERR(ptr);
489 		DBG("failed to map: %d", ret);
490 		return ret;
491 	}
492 
493 	for (i = 0; i < nr_relocs; i++) {
494 		struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
495 		uint32_t off;
496 		uint64_t iova;
497 		bool valid;
498 
499 		if (submit_reloc.submit_offset % 4) {
500 			DRM_ERROR("non-aligned reloc offset: %u\n",
501 					submit_reloc.submit_offset);
502 			ret = -EINVAL;
503 			goto out;
504 		}
505 
506 		/* offset in dwords: */
507 		off = submit_reloc.submit_offset / 4;
508 
509 		if ((off >= (obj->size / 4)) ||
510 				(off < last_offset)) {
511 			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
512 			ret = -EINVAL;
513 			goto out;
514 		}
515 
516 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
517 		if (ret)
518 			goto out;
519 
520 		if (valid)
521 			continue;
522 
523 		iova += submit_reloc.reloc_offset;
524 
525 		if (submit_reloc.shift < 0)
526 			iova >>= -submit_reloc.shift;
527 		else
528 			iova <<= submit_reloc.shift;
529 
530 		ptr[off] = iova | submit_reloc.or;
531 
532 		last_offset = off;
533 	}
534 
535 out:
536 	msm_gem_put_vaddr_locked(obj);
537 
538 	return ret;
539 }
540 
541 /* Cleanup submit at end of ioctl.  In the error case, this also drops
542  * references, unpins, and drops active refcnt.  In the non-error case,
543  * this is done when the submit is retired.
544  */
545 static void submit_cleanup(struct msm_gem_submit *submit, bool error)
546 {
547 	unsigned cleanup_flags = BO_LOCKED;
548 	unsigned i;
549 
550 	if (error)
551 		cleanup_flags |= BO_PINNED;
552 
553 	for (i = 0; i < submit->nr_bos; i++) {
554 		struct drm_gem_object *obj = submit->bos[i].obj;
555 		submit_cleanup_bo(submit, i, cleanup_flags);
556 		if (error)
557 			drm_gem_object_put(obj);
558 	}
559 }
560 
561 void msm_submit_retire(struct msm_gem_submit *submit)
562 {
563 	int i;
564 
565 	for (i = 0; i < submit->nr_bos; i++) {
566 		struct drm_gem_object *obj = submit->bos[i].obj;
567 
568 		drm_gem_object_put(obj);
569 	}
570 }
571 
572 struct msm_submit_post_dep {
573 	struct drm_syncobj *syncobj;
574 	uint64_t point;
575 	struct dma_fence_chain *chain;
576 };
577 
578 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
579                                            struct drm_file *file,
580                                            uint64_t in_syncobjs_addr,
581                                            uint32_t nr_in_syncobjs,
582                                            size_t syncobj_stride)
583 {
584 	struct drm_syncobj **syncobjs = NULL;
585 	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
586 	int ret = 0;
587 	uint32_t i, j;
588 
589 	syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
590 	                   GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
591 	if (!syncobjs)
592 		return ERR_PTR(-ENOMEM);
593 
594 	for (i = 0; i < nr_in_syncobjs; ++i) {
595 		uint64_t address = in_syncobjs_addr + i * syncobj_stride;
596 
597 		if (copy_from_user(&syncobj_desc,
598 			           u64_to_user_ptr(address),
599 			           min(syncobj_stride, sizeof(syncobj_desc)))) {
600 			ret = -EFAULT;
601 			break;
602 		}
603 
604 		if (syncobj_desc.point &&
605 		    !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) {
606 			ret = -EOPNOTSUPP;
607 			break;
608 		}
609 
610 		if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
611 			ret = -EINVAL;
612 			break;
613 		}
614 
615 		ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,
616 							   syncobj_desc.handle, syncobj_desc.point);
617 		if (ret)
618 			break;
619 
620 		if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
621 			syncobjs[i] =
622 				drm_syncobj_find(file, syncobj_desc.handle);
623 			if (!syncobjs[i]) {
624 				ret = -EINVAL;
625 				break;
626 			}
627 		}
628 	}
629 
630 	if (ret) {
631 		for (j = 0; j <= i; ++j) {
632 			if (syncobjs[j])
633 				drm_syncobj_put(syncobjs[j]);
634 		}
635 		kfree(syncobjs);
636 		return ERR_PTR(ret);
637 	}
638 	return syncobjs;
639 }
640 
641 static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
642                                uint32_t nr_syncobjs)
643 {
644 	uint32_t i;
645 
646 	for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
647 		if (syncobjs[i])
648 			drm_syncobj_replace_fence(syncobjs[i], NULL);
649 	}
650 }
651 
652 static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
653                                                        struct drm_file *file,
654                                                        uint64_t syncobjs_addr,
655                                                        uint32_t nr_syncobjs,
656                                                        size_t syncobj_stride)
657 {
658 	struct msm_submit_post_dep *post_deps;
659 	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
660 	int ret = 0;
661 	uint32_t i, j;
662 
663 	post_deps = kcalloc(nr_syncobjs, sizeof(*post_deps),
664 			    GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
665 	if (!post_deps)
666 		return ERR_PTR(-ENOMEM);
667 
668 	for (i = 0; i < nr_syncobjs; ++i) {
669 		uint64_t address = syncobjs_addr + i * syncobj_stride;
670 
671 		if (copy_from_user(&syncobj_desc,
672 			           u64_to_user_ptr(address),
673 			           min(syncobj_stride, sizeof(syncobj_desc)))) {
674 			ret = -EFAULT;
675 			break;
676 		}
677 
678 		post_deps[i].point = syncobj_desc.point;
679 
680 		if (syncobj_desc.flags) {
681 			ret = -EINVAL;
682 			break;
683 		}
684 
685 		if (syncobj_desc.point) {
686 			if (!drm_core_check_feature(dev,
687 			                            DRIVER_SYNCOBJ_TIMELINE)) {
688 				ret = -EOPNOTSUPP;
689 				break;
690 			}
691 
692 			post_deps[i].chain = dma_fence_chain_alloc();
693 			if (!post_deps[i].chain) {
694 				ret = -ENOMEM;
695 				break;
696 			}
697 		}
698 
699 		post_deps[i].syncobj =
700 			drm_syncobj_find(file, syncobj_desc.handle);
701 		if (!post_deps[i].syncobj) {
702 			ret = -EINVAL;
703 			break;
704 		}
705 	}
706 
707 	if (ret) {
708 		for (j = 0; j <= i; ++j) {
709 			dma_fence_chain_free(post_deps[j].chain);
710 			if (post_deps[j].syncobj)
711 				drm_syncobj_put(post_deps[j].syncobj);
712 		}
713 
714 		kfree(post_deps);
715 		return ERR_PTR(ret);
716 	}
717 
718 	return post_deps;
719 }
720 
721 static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
722                                   uint32_t count, struct dma_fence *fence)
723 {
724 	uint32_t i;
725 
726 	for (i = 0; post_deps && i < count; ++i) {
727 		if (post_deps[i].chain) {
728 			drm_syncobj_add_point(post_deps[i].syncobj,
729 			                      post_deps[i].chain,
730 			                      fence, post_deps[i].point);
731 			post_deps[i].chain = NULL;
732 		} else {
733 			drm_syncobj_replace_fence(post_deps[i].syncobj,
734 			                          fence);
735 		}
736 	}
737 }
738 
739 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
740 		struct drm_file *file)
741 {
742 	struct msm_drm_private *priv = dev->dev_private;
743 	struct drm_msm_gem_submit *args = data;
744 	struct msm_file_private *ctx = file->driver_priv;
745 	struct msm_gem_submit *submit = NULL;
746 	struct msm_gpu *gpu = priv->gpu;
747 	struct msm_gpu_submitqueue *queue;
748 	struct msm_ringbuffer *ring;
749 	struct msm_submit_post_dep *post_deps = NULL;
750 	struct drm_syncobj **syncobjs_to_reset = NULL;
751 	int out_fence_fd = -1;
752 	bool has_ww_ticket = false;
753 	unsigned i;
754 	int ret;
755 
756 	if (!gpu)
757 		return -ENXIO;
758 
759 	if (args->pad)
760 		return -EINVAL;
761 
762 	if (unlikely(!ctx->aspace) && !capable(CAP_SYS_RAWIO)) {
763 		DRM_ERROR_RATELIMITED("IOMMU support or CAP_SYS_RAWIO required!\n");
764 		return -EPERM;
765 	}
766 
767 	/* for now, we just have 3d pipe.. eventually this would need to
768 	 * be more clever to dispatch to appropriate gpu module:
769 	 */
770 	if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
771 		return -EINVAL;
772 
773 	if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
774 		return -EINVAL;
775 
776 	if (args->flags & MSM_SUBMIT_SUDO) {
777 		if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
778 		    !capable(CAP_SYS_RAWIO))
779 			return -EINVAL;
780 	}
781 
782 	queue = msm_submitqueue_get(ctx, args->queueid);
783 	if (!queue)
784 		return -ENOENT;
785 
786 	ring = gpu->rb[queue->ring_nr];
787 
788 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
789 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
790 		if (out_fence_fd < 0) {
791 			ret = out_fence_fd;
792 			goto out_post_unlock;
793 		}
794 	}
795 
796 	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
797 	if (IS_ERR(submit)) {
798 		ret = PTR_ERR(submit);
799 		goto out_post_unlock;
800 	}
801 
802 	trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
803 		args->nr_bos, args->nr_cmds);
804 
805 	ret = mutex_lock_interruptible(&queue->lock);
806 	if (ret)
807 		goto out_post_unlock;
808 
809 	if (args->flags & MSM_SUBMIT_SUDO)
810 		submit->in_rb = true;
811 
812 	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
813 		struct dma_fence *in_fence;
814 
815 		in_fence = sync_file_get_fence(args->fence_fd);
816 
817 		if (!in_fence) {
818 			ret = -EINVAL;
819 			goto out_unlock;
820 		}
821 
822 		ret = drm_sched_job_add_dependency(&submit->base, in_fence);
823 		if (ret)
824 			goto out_unlock;
825 	}
826 
827 	if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
828 		syncobjs_to_reset = msm_parse_deps(submit, file,
829 		                                   args->in_syncobjs,
830 		                                   args->nr_in_syncobjs,
831 		                                   args->syncobj_stride);
832 		if (IS_ERR(syncobjs_to_reset)) {
833 			ret = PTR_ERR(syncobjs_to_reset);
834 			goto out_unlock;
835 		}
836 	}
837 
838 	if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
839 		post_deps = msm_parse_post_deps(dev, file,
840 		                                args->out_syncobjs,
841 		                                args->nr_out_syncobjs,
842 		                                args->syncobj_stride);
843 		if (IS_ERR(post_deps)) {
844 			ret = PTR_ERR(post_deps);
845 			goto out_unlock;
846 		}
847 	}
848 
849 	ret = submit_lookup_objects(submit, args, file);
850 	if (ret)
851 		goto out;
852 
853 	ret = submit_lookup_cmds(submit, args, file);
854 	if (ret)
855 		goto out;
856 
857 	/* copy_*_user while holding a ww ticket upsets lockdep */
858 	ww_acquire_init(&submit->ticket, &reservation_ww_class);
859 	has_ww_ticket = true;
860 	ret = submit_lock_objects(submit);
861 	if (ret)
862 		goto out;
863 
864 	ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT));
865 	if (ret)
866 		goto out;
867 
868 	ret = submit_pin_objects(submit);
869 	if (ret)
870 		goto out;
871 
872 	for (i = 0; i < args->nr_cmds; i++) {
873 		struct drm_gem_object *obj;
874 		uint64_t iova;
875 
876 		ret = submit_bo(submit, submit->cmd[i].idx,
877 				&obj, &iova, NULL);
878 		if (ret)
879 			goto out;
880 
881 		if (!submit->cmd[i].size ||
882 			((submit->cmd[i].size + submit->cmd[i].offset) >
883 				obj->size / 4)) {
884 			DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4);
885 			ret = -EINVAL;
886 			goto out;
887 		}
888 
889 		submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
890 
891 		if (submit->valid)
892 			continue;
893 
894 		if (!gpu->allow_relocs) {
895 			if (submit->cmd[i].nr_relocs) {
896 				DRM_ERROR("relocs not allowed\n");
897 				ret = -EINVAL;
898 				goto out;
899 			}
900 
901 			continue;
902 		}
903 
904 		ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,
905 				submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
906 		if (ret)
907 			goto out;
908 	}
909 
910 	submit->nr_cmds = i;
911 
912 	idr_preload(GFP_KERNEL);
913 
914 	spin_lock(&queue->idr_lock);
915 
916 	/*
917 	 * If using userspace provided seqno fence, validate that the id
918 	 * is available before arming sched job.  Since access to fence_idr
919 	 * is serialized on the queue lock, the slot should be still avail
920 	 * after the job is armed
921 	 */
922 	if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
923 			(!args->fence || idr_find(&queue->fence_idr, args->fence))) {
924 		spin_unlock(&queue->idr_lock);
925 		idr_preload_end();
926 		ret = -EINVAL;
927 		goto out;
928 	}
929 
930 	drm_sched_job_arm(&submit->base);
931 
932 	submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
933 
934 	if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
935 		/*
936 		 * Userspace has assigned the seqno fence that it wants
937 		 * us to use.  It is an error to pick a fence sequence
938 		 * number that is not available.
939 		 */
940 		submit->fence_id = args->fence;
941 		ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
942 				    &submit->fence_id, submit->fence_id,
943 				    GFP_NOWAIT);
944 		/*
945 		 * We've already validated that the fence_id slot is valid,
946 		 * so if idr_alloc_u32 failed, it is a kernel bug
947 		 */
948 		WARN_ON(ret);
949 	} else {
950 		/*
951 		 * Allocate an id which can be used by WAIT_FENCE ioctl to map
952 		 * back to the underlying fence.
953 		 */
954 		submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
955 						    submit->user_fence, 1,
956 						    INT_MAX, GFP_NOWAIT);
957 	}
958 
959 	spin_unlock(&queue->idr_lock);
960 	idr_preload_end();
961 
962 	if (submit->fence_id < 0) {
963 		ret = submit->fence_id;
964 		submit->fence_id = 0;
965 	}
966 
967 	if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
968 		struct sync_file *sync_file = sync_file_create(submit->user_fence);
969 		if (!sync_file) {
970 			ret = -ENOMEM;
971 		} else {
972 			fd_install(out_fence_fd, sync_file->file);
973 			args->fence_fd = out_fence_fd;
974 		}
975 	}
976 
977 	submit_attach_object_fences(submit);
978 
979 	/* The scheduler owns a ref now: */
980 	msm_gem_submit_get(submit);
981 
982 	msm_rd_dump_submit(priv->rd, submit, NULL);
983 
984 	drm_sched_entity_push_job(&submit->base);
985 
986 	args->fence = submit->fence_id;
987 	queue->last_fence = submit->fence_id;
988 
989 	msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
990 	msm_process_post_deps(post_deps, args->nr_out_syncobjs,
991 	                      submit->user_fence);
992 
993 
994 out:
995 	submit_cleanup(submit, !!ret);
996 	if (has_ww_ticket)
997 		ww_acquire_fini(&submit->ticket);
998 out_unlock:
999 	mutex_unlock(&queue->lock);
1000 out_post_unlock:
1001 	if (ret && (out_fence_fd >= 0))
1002 		put_unused_fd(out_fence_fd);
1003 
1004 	if (!IS_ERR_OR_NULL(submit)) {
1005 		msm_gem_submit_put(submit);
1006 	} else {
1007 		/*
1008 		 * If the submit hasn't yet taken ownership of the queue
1009 		 * then we need to drop the reference ourself:
1010 		 */
1011 		msm_submitqueue_put(queue);
1012 	}
1013 	if (!IS_ERR_OR_NULL(post_deps)) {
1014 		for (i = 0; i < args->nr_out_syncobjs; ++i) {
1015 			kfree(post_deps[i].chain);
1016 			drm_syncobj_put(post_deps[i].syncobj);
1017 		}
1018 		kfree(post_deps);
1019 	}
1020 
1021 	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
1022 		for (i = 0; i < args->nr_in_syncobjs; ++i) {
1023 			if (syncobjs_to_reset[i])
1024 				drm_syncobj_put(syncobjs_to_reset[i]);
1025 		}
1026 		kfree(syncobjs_to_reset);
1027 	}
1028 
1029 	return ret;
1030 }
1031