1 /*
2  * Copyright 2008 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22  * DEALINGS IN THE SOFTWARE.
23  *
24  * Authors:
25  *    Jerome Glisse <glisse@freedesktop.org>
26  */
27 
28 #include <linux/file.h>
29 #include <linux/pagemap.h>
30 #include <linux/sync_file.h>
31 #include <linux/dma-buf.h>
32 
33 #include <drm/amdgpu_drm.h>
34 #include <drm/drm_syncobj.h>
35 #include <drm/ttm/ttm_tt.h>
36 
37 #include "amdgpu_cs.h"
38 #include "amdgpu.h"
39 #include "amdgpu_trace.h"
40 #include "amdgpu_gmc.h"
41 #include "amdgpu_gem.h"
42 #include "amdgpu_ras.h"
43 
44 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,
45 				 struct amdgpu_device *adev,
46 				 struct drm_file *filp,
47 				 union drm_amdgpu_cs *cs)
48 {
49 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
50 
51 	if (cs->in.num_chunks == 0)
52 		return -EINVAL;
53 
54 	memset(p, 0, sizeof(*p));
55 	p->adev = adev;
56 	p->filp = filp;
57 
58 	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
59 	if (!p->ctx)
60 		return -EINVAL;
61 
62 	if (atomic_read(&p->ctx->guilty)) {
63 		amdgpu_ctx_put(p->ctx);
64 		return -ECANCELED;
65 	}
66 
67 	amdgpu_sync_create(&p->sync);
68 	drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
69 		      DRM_EXEC_IGNORE_DUPLICATES);
70 	return 0;
71 }
72 
73 static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
74 			     struct drm_amdgpu_cs_chunk_ib *chunk_ib)
75 {
76 	struct drm_sched_entity *entity;
77 	unsigned int i;
78 	int r;
79 
80 	r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,
81 				  chunk_ib->ip_instance,
82 				  chunk_ib->ring, &entity);
83 	if (r)
84 		return r;
85 
86 	/*
87 	 * Abort if there is no run queue associated with this entity.
88 	 * Possibly because of disabled HW IP.
89 	 */
90 	if (entity->rq == NULL)
91 		return -EINVAL;
92 
93 	/* Check if we can add this IB to some existing job */
94 	for (i = 0; i < p->gang_size; ++i)
95 		if (p->entities[i] == entity)
96 			return i;
97 
98 	/* If not increase the gang size if possible */
99 	if (i == AMDGPU_CS_GANG_SIZE)
100 		return -EINVAL;
101 
102 	p->entities[i] = entity;
103 	p->gang_size = i + 1;
104 	return i;
105 }
106 
107 static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
108 			   struct drm_amdgpu_cs_chunk_ib *chunk_ib,
109 			   unsigned int *num_ibs)
110 {
111 	int r;
112 
113 	r = amdgpu_cs_job_idx(p, chunk_ib);
114 	if (r < 0)
115 		return r;
116 
117 	if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
118 		return -EINVAL;
119 
120 	++(num_ibs[r]);
121 	p->gang_leader_idx = r;
122 	return 0;
123 }
124 
125 static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,
126 				   struct drm_amdgpu_cs_chunk_fence *data,
127 				   uint32_t *offset)
128 {
129 	struct drm_gem_object *gobj;
130 	unsigned long size;
131 
132 	gobj = drm_gem_object_lookup(p->filp, data->handle);
133 	if (gobj == NULL)
134 		return -EINVAL;
135 
136 	p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
137 	drm_gem_object_put(gobj);
138 
139 	size = amdgpu_bo_size(p->uf_bo);
140 	if (size != PAGE_SIZE || data->offset > (size - 8))
141 		return -EINVAL;
142 
143 	if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm))
144 		return -EINVAL;
145 
146 	*offset = data->offset;
147 	return 0;
148 }
149 
150 static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
151 				   struct drm_amdgpu_bo_list_in *data)
152 {
153 	struct drm_amdgpu_bo_list_entry *info;
154 	int r;
155 
156 	r = amdgpu_bo_create_list_entry_array(data, &info);
157 	if (r)
158 		return r;
159 
160 	r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
161 				  &p->bo_list);
162 	if (r)
163 		goto error_free;
164 
165 	kvfree(info);
166 	return 0;
167 
168 error_free:
169 	kvfree(info);
170 
171 	return r;
172 }
173 
174 /* Copy the data from userspace and go over it the first time */
175 static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
176 			   union drm_amdgpu_cs *cs)
177 {
178 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
179 	unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };
180 	struct amdgpu_vm *vm = &fpriv->vm;
181 	uint64_t *chunk_array_user;
182 	uint64_t *chunk_array;
183 	uint32_t uf_offset = 0;
184 	size_t size;
185 	int ret;
186 	int i;
187 
188 	chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t),
189 				     GFP_KERNEL);
190 	if (!chunk_array)
191 		return -ENOMEM;
192 
193 	/* get chunks */
194 	chunk_array_user = u64_to_user_ptr(cs->in.chunks);
195 	if (copy_from_user(chunk_array, chunk_array_user,
196 			   sizeof(uint64_t)*cs->in.num_chunks)) {
197 		ret = -EFAULT;
198 		goto free_chunk;
199 	}
200 
201 	p->nchunks = cs->in.num_chunks;
202 	p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
203 			    GFP_KERNEL);
204 	if (!p->chunks) {
205 		ret = -ENOMEM;
206 		goto free_chunk;
207 	}
208 
209 	for (i = 0; i < p->nchunks; i++) {
210 		struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL;
211 		struct drm_amdgpu_cs_chunk user_chunk;
212 		uint32_t __user *cdata;
213 
214 		chunk_ptr = u64_to_user_ptr(chunk_array[i]);
215 		if (copy_from_user(&user_chunk, chunk_ptr,
216 				       sizeof(struct drm_amdgpu_cs_chunk))) {
217 			ret = -EFAULT;
218 			i--;
219 			goto free_partial_kdata;
220 		}
221 		p->chunks[i].chunk_id = user_chunk.chunk_id;
222 		p->chunks[i].length_dw = user_chunk.length_dw;
223 
224 		size = p->chunks[i].length_dw;
225 		cdata = u64_to_user_ptr(user_chunk.chunk_data);
226 
227 		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t),
228 						    GFP_KERNEL);
229 		if (p->chunks[i].kdata == NULL) {
230 			ret = -ENOMEM;
231 			i--;
232 			goto free_partial_kdata;
233 		}
234 		size *= sizeof(uint32_t);
235 		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
236 			ret = -EFAULT;
237 			goto free_partial_kdata;
238 		}
239 
240 		/* Assume the worst on the following checks */
241 		ret = -EINVAL;
242 		switch (p->chunks[i].chunk_id) {
243 		case AMDGPU_CHUNK_ID_IB:
244 			if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))
245 				goto free_partial_kdata;
246 
247 			ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);
248 			if (ret)
249 				goto free_partial_kdata;
250 			break;
251 
252 		case AMDGPU_CHUNK_ID_FENCE:
253 			if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))
254 				goto free_partial_kdata;
255 
256 			ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,
257 						      &uf_offset);
258 			if (ret)
259 				goto free_partial_kdata;
260 			break;
261 
262 		case AMDGPU_CHUNK_ID_BO_HANDLES:
263 			if (size < sizeof(struct drm_amdgpu_bo_list_in))
264 				goto free_partial_kdata;
265 
266 			ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);
267 			if (ret)
268 				goto free_partial_kdata;
269 			break;
270 
271 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
272 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
273 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
274 		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
275 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
276 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
277 		case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
278 			break;
279 
280 		default:
281 			goto free_partial_kdata;
282 		}
283 	}
284 
285 	if (!p->gang_size) {
286 		ret = -EINVAL;
287 		goto free_all_kdata;
288 	}
289 
290 	for (i = 0; i < p->gang_size; ++i) {
291 		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
292 				       num_ibs[i], &p->jobs[i]);
293 		if (ret)
294 			goto free_all_kdata;
295 	}
296 	p->gang_leader = p->jobs[p->gang_leader_idx];
297 
298 	if (p->ctx->generation != p->gang_leader->generation) {
299 		ret = -ECANCELED;
300 		goto free_all_kdata;
301 	}
302 
303 	if (p->uf_bo)
304 		p->gang_leader->uf_addr = uf_offset;
305 	kvfree(chunk_array);
306 
307 	/* Use this opportunity to fill in task info for the vm */
308 	amdgpu_vm_set_task_info(vm);
309 
310 	return 0;
311 
312 free_all_kdata:
313 	i = p->nchunks - 1;
314 free_partial_kdata:
315 	for (; i >= 0; i--)
316 		kvfree(p->chunks[i].kdata);
317 	kvfree(p->chunks);
318 	p->chunks = NULL;
319 	p->nchunks = 0;
320 free_chunk:
321 	kvfree(chunk_array);
322 
323 	return ret;
324 }
325 
326 static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
327 			   struct amdgpu_cs_chunk *chunk,
328 			   unsigned int *ce_preempt,
329 			   unsigned int *de_preempt)
330 {
331 	struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;
332 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
333 	struct amdgpu_vm *vm = &fpriv->vm;
334 	struct amdgpu_ring *ring;
335 	struct amdgpu_job *job;
336 	struct amdgpu_ib *ib;
337 	int r;
338 
339 	r = amdgpu_cs_job_idx(p, chunk_ib);
340 	if (r < 0)
341 		return r;
342 
343 	job = p->jobs[r];
344 	ring = amdgpu_job_ring(job);
345 	ib = &job->ibs[job->num_ibs++];
346 
347 	/* MM engine doesn't support user fences */
348 	if (p->uf_bo && ring->funcs->no_user_fence)
349 		return -EINVAL;
350 
351 	if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
352 	    chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
353 		if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
354 			(*ce_preempt)++;
355 		else
356 			(*de_preempt)++;
357 
358 		/* Each GFX command submit allows only 1 IB max
359 		 * preemptible for CE & DE */
360 		if (*ce_preempt > 1 || *de_preempt > 1)
361 			return -EINVAL;
362 	}
363 
364 	if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
365 		job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
366 
367 	r =  amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?
368 			   chunk_ib->ib_bytes : 0,
369 			   AMDGPU_IB_POOL_DELAYED, ib);
370 	if (r) {
371 		DRM_ERROR("Failed to get ib !\n");
372 		return r;
373 	}
374 
375 	ib->gpu_addr = chunk_ib->va_start;
376 	ib->length_dw = chunk_ib->ib_bytes / 4;
377 	ib->flags = chunk_ib->flags;
378 	return 0;
379 }
380 
381 static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
382 				     struct amdgpu_cs_chunk *chunk)
383 {
384 	struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;
385 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
386 	unsigned int num_deps;
387 	int i, r;
388 
389 	num_deps = chunk->length_dw * 4 /
390 		sizeof(struct drm_amdgpu_cs_chunk_dep);
391 
392 	for (i = 0; i < num_deps; ++i) {
393 		struct amdgpu_ctx *ctx;
394 		struct drm_sched_entity *entity;
395 		struct dma_fence *fence;
396 
397 		ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
398 		if (ctx == NULL)
399 			return -EINVAL;
400 
401 		r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
402 					  deps[i].ip_instance,
403 					  deps[i].ring, &entity);
404 		if (r) {
405 			amdgpu_ctx_put(ctx);
406 			return r;
407 		}
408 
409 		fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
410 		amdgpu_ctx_put(ctx);
411 
412 		if (IS_ERR(fence))
413 			return PTR_ERR(fence);
414 		else if (!fence)
415 			continue;
416 
417 		if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
418 			struct drm_sched_fence *s_fence;
419 			struct dma_fence *old = fence;
420 
421 			s_fence = to_drm_sched_fence(fence);
422 			fence = dma_fence_get(&s_fence->scheduled);
423 			dma_fence_put(old);
424 		}
425 
426 		r = amdgpu_sync_fence(&p->sync, fence);
427 		dma_fence_put(fence);
428 		if (r)
429 			return r;
430 	}
431 	return 0;
432 }
433 
434 static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
435 					 uint32_t handle, u64 point,
436 					 u64 flags)
437 {
438 	struct dma_fence *fence;
439 	int r;
440 
441 	r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
442 	if (r) {
443 		DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
444 			  handle, point, r);
445 		return r;
446 	}
447 
448 	r = amdgpu_sync_fence(&p->sync, fence);
449 	dma_fence_put(fence);
450 	return r;
451 }
452 
453 static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,
454 				   struct amdgpu_cs_chunk *chunk)
455 {
456 	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
457 	unsigned int num_deps;
458 	int i, r;
459 
460 	num_deps = chunk->length_dw * 4 /
461 		sizeof(struct drm_amdgpu_cs_chunk_sem);
462 	for (i = 0; i < num_deps; ++i) {
463 		r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);
464 		if (r)
465 			return r;
466 	}
467 
468 	return 0;
469 }
470 
471 static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,
472 					      struct amdgpu_cs_chunk *chunk)
473 {
474 	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
475 	unsigned int num_deps;
476 	int i, r;
477 
478 	num_deps = chunk->length_dw * 4 /
479 		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
480 	for (i = 0; i < num_deps; ++i) {
481 		r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,
482 						  syncobj_deps[i].point,
483 						  syncobj_deps[i].flags);
484 		if (r)
485 			return r;
486 	}
487 
488 	return 0;
489 }
490 
491 static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,
492 				    struct amdgpu_cs_chunk *chunk)
493 {
494 	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
495 	unsigned int num_deps;
496 	int i;
497 
498 	num_deps = chunk->length_dw * 4 /
499 		sizeof(struct drm_amdgpu_cs_chunk_sem);
500 
501 	if (p->post_deps)
502 		return -EINVAL;
503 
504 	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
505 				     GFP_KERNEL);
506 	p->num_post_deps = 0;
507 
508 	if (!p->post_deps)
509 		return -ENOMEM;
510 
511 
512 	for (i = 0; i < num_deps; ++i) {
513 		p->post_deps[i].syncobj =
514 			drm_syncobj_find(p->filp, deps[i].handle);
515 		if (!p->post_deps[i].syncobj)
516 			return -EINVAL;
517 		p->post_deps[i].chain = NULL;
518 		p->post_deps[i].point = 0;
519 		p->num_post_deps++;
520 	}
521 
522 	return 0;
523 }
524 
525 static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,
526 						struct amdgpu_cs_chunk *chunk)
527 {
528 	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
529 	unsigned int num_deps;
530 	int i;
531 
532 	num_deps = chunk->length_dw * 4 /
533 		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
534 
535 	if (p->post_deps)
536 		return -EINVAL;
537 
538 	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
539 				     GFP_KERNEL);
540 	p->num_post_deps = 0;
541 
542 	if (!p->post_deps)
543 		return -ENOMEM;
544 
545 	for (i = 0; i < num_deps; ++i) {
546 		struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
547 
548 		dep->chain = NULL;
549 		if (syncobj_deps[i].point) {
550 			dep->chain = dma_fence_chain_alloc();
551 			if (!dep->chain)
552 				return -ENOMEM;
553 		}
554 
555 		dep->syncobj = drm_syncobj_find(p->filp,
556 						syncobj_deps[i].handle);
557 		if (!dep->syncobj) {
558 			dma_fence_chain_free(dep->chain);
559 			return -EINVAL;
560 		}
561 		dep->point = syncobj_deps[i].point;
562 		p->num_post_deps++;
563 	}
564 
565 	return 0;
566 }
567 
568 static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p,
569 			       struct amdgpu_cs_chunk *chunk)
570 {
571 	struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata;
572 	int i;
573 
574 	if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW)
575 		return -EINVAL;
576 
577 	for (i = 0; i < p->gang_size; ++i) {
578 		p->jobs[i]->shadow_va = shadow->shadow_va;
579 		p->jobs[i]->csa_va = shadow->csa_va;
580 		p->jobs[i]->gds_va = shadow->gds_va;
581 		p->jobs[i]->init_shadow =
582 			shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW;
583 	}
584 
585 	return 0;
586 }
587 
588 static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)
589 {
590 	unsigned int ce_preempt = 0, de_preempt = 0;
591 	int i, r;
592 
593 	for (i = 0; i < p->nchunks; ++i) {
594 		struct amdgpu_cs_chunk *chunk;
595 
596 		chunk = &p->chunks[i];
597 
598 		switch (chunk->chunk_id) {
599 		case AMDGPU_CHUNK_ID_IB:
600 			r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);
601 			if (r)
602 				return r;
603 			break;
604 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
605 		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
606 			r = amdgpu_cs_p2_dependencies(p, chunk);
607 			if (r)
608 				return r;
609 			break;
610 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
611 			r = amdgpu_cs_p2_syncobj_in(p, chunk);
612 			if (r)
613 				return r;
614 			break;
615 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
616 			r = amdgpu_cs_p2_syncobj_out(p, chunk);
617 			if (r)
618 				return r;
619 			break;
620 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
621 			r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);
622 			if (r)
623 				return r;
624 			break;
625 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
626 			r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);
627 			if (r)
628 				return r;
629 			break;
630 		case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
631 			r = amdgpu_cs_p2_shadow(p, chunk);
632 			if (r)
633 				return r;
634 			break;
635 		}
636 	}
637 
638 	return 0;
639 }
640 
641 /* Convert microseconds to bytes. */
642 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
643 {
644 	if (us <= 0 || !adev->mm_stats.log2_max_MBps)
645 		return 0;
646 
647 	/* Since accum_us is incremented by a million per second, just
648 	 * multiply it by the number of MB/s to get the number of bytes.
649 	 */
650 	return us << adev->mm_stats.log2_max_MBps;
651 }
652 
653 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
654 {
655 	if (!adev->mm_stats.log2_max_MBps)
656 		return 0;
657 
658 	return bytes >> adev->mm_stats.log2_max_MBps;
659 }
660 
661 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
662  * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
663  * which means it can go over the threshold once. If that happens, the driver
664  * will be in debt and no other buffer migrations can be done until that debt
665  * is repaid.
666  *
667  * This approach allows moving a buffer of any size (it's important to allow
668  * that).
669  *
670  * The currency is simply time in microseconds and it increases as the clock
671  * ticks. The accumulated microseconds (us) are converted to bytes and
672  * returned.
673  */
674 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
675 					      u64 *max_bytes,
676 					      u64 *max_vis_bytes)
677 {
678 	s64 time_us, increment_us;
679 	u64 free_vram, total_vram, used_vram;
680 	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
681 	 * throttling.
682 	 *
683 	 * It means that in order to get full max MBps, at least 5 IBs per
684 	 * second must be submitted and not more than 200ms apart from each
685 	 * other.
686 	 */
687 	const s64 us_upper_bound = 200000;
688 
689 	if (!adev->mm_stats.log2_max_MBps) {
690 		*max_bytes = 0;
691 		*max_vis_bytes = 0;
692 		return;
693 	}
694 
695 	total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
696 	used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager);
697 	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
698 
699 	spin_lock(&adev->mm_stats.lock);
700 
701 	/* Increase the amount of accumulated us. */
702 	time_us = ktime_to_us(ktime_get());
703 	increment_us = time_us - adev->mm_stats.last_update_us;
704 	adev->mm_stats.last_update_us = time_us;
705 	adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
706 				      us_upper_bound);
707 
708 	/* This prevents the short period of low performance when the VRAM
709 	 * usage is low and the driver is in debt or doesn't have enough
710 	 * accumulated us to fill VRAM quickly.
711 	 *
712 	 * The situation can occur in these cases:
713 	 * - a lot of VRAM is freed by userspace
714 	 * - the presence of a big buffer causes a lot of evictions
715 	 *   (solution: split buffers into smaller ones)
716 	 *
717 	 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
718 	 * accum_us to a positive number.
719 	 */
720 	if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
721 		s64 min_us;
722 
723 		/* Be more aggressive on dGPUs. Try to fill a portion of free
724 		 * VRAM now.
725 		 */
726 		if (!(adev->flags & AMD_IS_APU))
727 			min_us = bytes_to_us(adev, free_vram / 4);
728 		else
729 			min_us = 0; /* Reset accum_us on APUs. */
730 
731 		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
732 	}
733 
734 	/* This is set to 0 if the driver is in debt to disallow (optional)
735 	 * buffer moves.
736 	 */
737 	*max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
738 
739 	/* Do the same for visible VRAM if half of it is free */
740 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
741 		u64 total_vis_vram = adev->gmc.visible_vram_size;
742 		u64 used_vis_vram =
743 		  amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
744 
745 		if (used_vis_vram < total_vis_vram) {
746 			u64 free_vis_vram = total_vis_vram - used_vis_vram;
747 
748 			adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
749 							  increment_us, us_upper_bound);
750 
751 			if (free_vis_vram >= total_vis_vram / 2)
752 				adev->mm_stats.accum_us_vis =
753 					max(bytes_to_us(adev, free_vis_vram / 2),
754 					    adev->mm_stats.accum_us_vis);
755 		}
756 
757 		*max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
758 	} else {
759 		*max_vis_bytes = 0;
760 	}
761 
762 	spin_unlock(&adev->mm_stats.lock);
763 }
764 
765 /* Report how many bytes have really been moved for the last command
766  * submission. This can result in a debt that can stop buffer migrations
767  * temporarily.
768  */
769 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
770 				  u64 num_vis_bytes)
771 {
772 	spin_lock(&adev->mm_stats.lock);
773 	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
774 	adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
775 	spin_unlock(&adev->mm_stats.lock);
776 }
777 
778 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)
779 {
780 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
781 	struct amdgpu_cs_parser *p = param;
782 	struct ttm_operation_ctx ctx = {
783 		.interruptible = true,
784 		.no_wait_gpu = false,
785 		.resv = bo->tbo.base.resv
786 	};
787 	uint32_t domain;
788 	int r;
789 
790 	if (bo->tbo.pin_count)
791 		return 0;
792 
793 	/* Don't move this buffer if we have depleted our allowance
794 	 * to move it. Don't move anything if the threshold is zero.
795 	 */
796 	if (p->bytes_moved < p->bytes_moved_threshold &&
797 	    (!bo->tbo.base.dma_buf ||
798 	    list_empty(&bo->tbo.base.dma_buf->attachments))) {
799 		if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
800 		    (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
801 			/* And don't move a CPU_ACCESS_REQUIRED BO to limited
802 			 * visible VRAM if we've depleted our allowance to do
803 			 * that.
804 			 */
805 			if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
806 				domain = bo->preferred_domains;
807 			else
808 				domain = bo->allowed_domains;
809 		} else {
810 			domain = bo->preferred_domains;
811 		}
812 	} else {
813 		domain = bo->allowed_domains;
814 	}
815 
816 retry:
817 	amdgpu_bo_placement_from_domain(bo, domain);
818 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
819 
820 	p->bytes_moved += ctx.bytes_moved;
821 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
822 	    amdgpu_bo_in_cpu_visible_vram(bo))
823 		p->bytes_moved_vis += ctx.bytes_moved;
824 
825 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
826 		domain = bo->allowed_domains;
827 		goto retry;
828 	}
829 
830 	return r;
831 }
832 
833 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
834 				union drm_amdgpu_cs *cs)
835 {
836 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
837 	struct ttm_operation_ctx ctx = { true, false };
838 	struct amdgpu_vm *vm = &fpriv->vm;
839 	struct amdgpu_bo_list_entry *e;
840 	struct drm_gem_object *obj;
841 	unsigned long index;
842 	unsigned int i;
843 	int r;
844 
845 	/* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
846 	if (cs->in.bo_list_handle) {
847 		if (p->bo_list)
848 			return -EINVAL;
849 
850 		r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
851 				       &p->bo_list);
852 		if (r)
853 			return r;
854 	} else if (!p->bo_list) {
855 		/* Create a empty bo_list when no handle is provided */
856 		r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
857 					  &p->bo_list);
858 		if (r)
859 			return r;
860 	}
861 
862 	mutex_lock(&p->bo_list->bo_list_mutex);
863 
864 	/* Get userptr backing pages. If pages are updated after registered
865 	 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
866 	 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
867 	 */
868 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
869 		bool userpage_invalidated = false;
870 		struct amdgpu_bo *bo = e->bo;
871 		int i;
872 
873 		e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
874 					sizeof(struct page *),
875 					GFP_KERNEL | __GFP_ZERO);
876 		if (!e->user_pages) {
877 			DRM_ERROR("kvmalloc_array failure\n");
878 			r = -ENOMEM;
879 			goto out_free_user_pages;
880 		}
881 
882 		r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages, &e->range);
883 		if (r) {
884 			kvfree(e->user_pages);
885 			e->user_pages = NULL;
886 			goto out_free_user_pages;
887 		}
888 
889 		for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
890 			if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
891 				userpage_invalidated = true;
892 				break;
893 			}
894 		}
895 		e->user_invalidated = userpage_invalidated;
896 	}
897 
898 	drm_exec_until_all_locked(&p->exec) {
899 		r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size);
900 		drm_exec_retry_on_contention(&p->exec);
901 		if (unlikely(r))
902 			goto out_free_user_pages;
903 
904 		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
905 			/* One fence for TTM and one for each CS job */
906 			r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base,
907 						 1 + p->gang_size);
908 			drm_exec_retry_on_contention(&p->exec);
909 			if (unlikely(r))
910 				goto out_free_user_pages;
911 
912 			e->bo_va = amdgpu_vm_bo_find(vm, e->bo);
913 		}
914 
915 		if (p->uf_bo) {
916 			r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base,
917 						 1 + p->gang_size);
918 			drm_exec_retry_on_contention(&p->exec);
919 			if (unlikely(r))
920 				goto out_free_user_pages;
921 		}
922 	}
923 
924 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
925 		struct mm_struct *usermm;
926 
927 		usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm);
928 		if (usermm && usermm != current->mm) {
929 			r = -EPERM;
930 			goto out_free_user_pages;
931 		}
932 
933 		if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) &&
934 		    e->user_invalidated && e->user_pages) {
935 			amdgpu_bo_placement_from_domain(e->bo,
936 							AMDGPU_GEM_DOMAIN_CPU);
937 			r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement,
938 					    &ctx);
939 			if (r)
940 				goto out_free_user_pages;
941 
942 			amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm,
943 						     e->user_pages);
944 		}
945 
946 		kvfree(e->user_pages);
947 		e->user_pages = NULL;
948 	}
949 
950 	amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
951 					  &p->bytes_moved_vis_threshold);
952 	p->bytes_moved = 0;
953 	p->bytes_moved_vis = 0;
954 
955 	r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
956 				      amdgpu_cs_bo_validate, p);
957 	if (r) {
958 		DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
959 		goto out_free_user_pages;
960 	}
961 
962 	drm_exec_for_each_locked_object(&p->exec, index, obj) {
963 		r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj));
964 		if (unlikely(r))
965 			goto out_free_user_pages;
966 	}
967 
968 	if (p->uf_bo) {
969 		r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo);
970 		if (unlikely(r))
971 			goto out_free_user_pages;
972 
973 		p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo);
974 	}
975 
976 	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
977 				     p->bytes_moved_vis);
978 
979 	for (i = 0; i < p->gang_size; ++i)
980 		amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,
981 					 p->bo_list->gws_obj,
982 					 p->bo_list->oa_obj);
983 	return 0;
984 
985 out_free_user_pages:
986 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
987 		struct amdgpu_bo *bo = e->bo;
988 
989 		if (!e->user_pages)
990 			continue;
991 		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, e->range);
992 		kvfree(e->user_pages);
993 		e->user_pages = NULL;
994 		e->range = NULL;
995 	}
996 	mutex_unlock(&p->bo_list->bo_list_mutex);
997 	return r;
998 }
999 
1000 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)
1001 {
1002 	int i, j;
1003 
1004 	if (!trace_amdgpu_cs_enabled())
1005 		return;
1006 
1007 	for (i = 0; i < p->gang_size; ++i) {
1008 		struct amdgpu_job *job = p->jobs[i];
1009 
1010 		for (j = 0; j < job->num_ibs; ++j)
1011 			trace_amdgpu_cs(p, job, &job->ibs[j]);
1012 	}
1013 }
1014 
1015 static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
1016 			       struct amdgpu_job *job)
1017 {
1018 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1019 	unsigned int i;
1020 	int r;
1021 
1022 	/* Only for UVD/VCE VM emulation */
1023 	if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)
1024 		return 0;
1025 
1026 	for (i = 0; i < job->num_ibs; ++i) {
1027 		struct amdgpu_ib *ib = &job->ibs[i];
1028 		struct amdgpu_bo_va_mapping *m;
1029 		struct amdgpu_bo *aobj;
1030 		uint64_t va_start;
1031 		uint8_t *kptr;
1032 
1033 		va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;
1034 		r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
1035 		if (r) {
1036 			DRM_ERROR("IB va_start is invalid\n");
1037 			return r;
1038 		}
1039 
1040 		if ((va_start + ib->length_dw * 4) >
1041 		    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
1042 			DRM_ERROR("IB va_start+ib_bytes is invalid\n");
1043 			return -EINVAL;
1044 		}
1045 
1046 		/* the IB should be reserved at this point */
1047 		r = amdgpu_bo_kmap(aobj, (void **)&kptr);
1048 		if (r)
1049 			return r;
1050 
1051 		kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);
1052 
1053 		if (ring->funcs->parse_cs) {
1054 			memcpy(ib->ptr, kptr, ib->length_dw * 4);
1055 			amdgpu_bo_kunmap(aobj);
1056 
1057 			r = amdgpu_ring_parse_cs(ring, p, job, ib);
1058 			if (r)
1059 				return r;
1060 		} else {
1061 			ib->ptr = (uint32_t *)kptr;
1062 			r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);
1063 			amdgpu_bo_kunmap(aobj);
1064 			if (r)
1065 				return r;
1066 		}
1067 	}
1068 
1069 	return 0;
1070 }
1071 
1072 static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)
1073 {
1074 	unsigned int i;
1075 	int r;
1076 
1077 	for (i = 0; i < p->gang_size; ++i) {
1078 		r = amdgpu_cs_patch_ibs(p, p->jobs[i]);
1079 		if (r)
1080 			return r;
1081 	}
1082 	return 0;
1083 }
1084 
1085 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
1086 {
1087 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1088 	struct amdgpu_job *job = p->gang_leader;
1089 	struct amdgpu_device *adev = p->adev;
1090 	struct amdgpu_vm *vm = &fpriv->vm;
1091 	struct amdgpu_bo_list_entry *e;
1092 	struct amdgpu_bo_va *bo_va;
1093 	unsigned int i;
1094 	int r;
1095 
1096 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
1097 	if (r)
1098 		return r;
1099 
1100 	r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1101 	if (r)
1102 		return r;
1103 
1104 	r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update);
1105 	if (r)
1106 		return r;
1107 
1108 	if (fpriv->csa_va) {
1109 		bo_va = fpriv->csa_va;
1110 		BUG_ON(!bo_va);
1111 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1112 		if (r)
1113 			return r;
1114 
1115 		r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
1116 		if (r)
1117 			return r;
1118 	}
1119 
1120 	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1121 		bo_va = e->bo_va;
1122 		if (bo_va == NULL)
1123 			continue;
1124 
1125 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1126 		if (r)
1127 			return r;
1128 
1129 		r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
1130 		if (r)
1131 			return r;
1132 	}
1133 
1134 	r = amdgpu_vm_handle_moved(adev, vm);
1135 	if (r)
1136 		return r;
1137 
1138 	r = amdgpu_vm_update_pdes(adev, vm, false);
1139 	if (r)
1140 		return r;
1141 
1142 	r = amdgpu_sync_fence(&p->sync, vm->last_update);
1143 	if (r)
1144 		return r;
1145 
1146 	for (i = 0; i < p->gang_size; ++i) {
1147 		job = p->jobs[i];
1148 
1149 		if (!job->vm)
1150 			continue;
1151 
1152 		job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
1153 	}
1154 
1155 	if (amdgpu_vm_debug) {
1156 		/* Invalidate all BOs to test for userspace bugs */
1157 		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1158 			struct amdgpu_bo *bo = e->bo;
1159 
1160 			/* ignore duplicates */
1161 			if (!bo)
1162 				continue;
1163 
1164 			amdgpu_vm_bo_invalidate(adev, bo, false);
1165 		}
1166 	}
1167 
1168 	return 0;
1169 }
1170 
1171 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
1172 {
1173 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1174 	struct drm_gpu_scheduler *sched;
1175 	struct drm_gem_object *obj;
1176 	struct dma_fence *fence;
1177 	unsigned long index;
1178 	unsigned int i;
1179 	int r;
1180 
1181 	r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]);
1182 	if (r) {
1183 		if (r != -ERESTARTSYS)
1184 			DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");
1185 		return r;
1186 	}
1187 
1188 	drm_exec_for_each_locked_object(&p->exec, index, obj) {
1189 		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
1190 
1191 		struct dma_resv *resv = bo->tbo.base.resv;
1192 		enum amdgpu_sync_mode sync_mode;
1193 
1194 		sync_mode = amdgpu_bo_explicit_sync(bo) ?
1195 			AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
1196 		r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode,
1197 				     &fpriv->vm);
1198 		if (r)
1199 			return r;
1200 	}
1201 
1202 	for (i = 0; i < p->gang_size; ++i) {
1203 		r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]);
1204 		if (r)
1205 			return r;
1206 	}
1207 
1208 	sched = p->gang_leader->base.entity->rq->sched;
1209 	while ((fence = amdgpu_sync_get_fence(&p->sync))) {
1210 		struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);
1211 
1212 		/*
1213 		 * When we have an dependency it might be necessary to insert a
1214 		 * pipeline sync to make sure that all caches etc are flushed and the
1215 		 * next job actually sees the results from the previous one
1216 		 * before we start executing on the same scheduler ring.
1217 		 */
1218 		if (!s_fence || s_fence->sched != sched) {
1219 			dma_fence_put(fence);
1220 			continue;
1221 		}
1222 
1223 		r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence);
1224 		dma_fence_put(fence);
1225 		if (r)
1226 			return r;
1227 	}
1228 	return 0;
1229 }
1230 
1231 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
1232 {
1233 	int i;
1234 
1235 	for (i = 0; i < p->num_post_deps; ++i) {
1236 		if (p->post_deps[i].chain && p->post_deps[i].point) {
1237 			drm_syncobj_add_point(p->post_deps[i].syncobj,
1238 					      p->post_deps[i].chain,
1239 					      p->fence, p->post_deps[i].point);
1240 			p->post_deps[i].chain = NULL;
1241 		} else {
1242 			drm_syncobj_replace_fence(p->post_deps[i].syncobj,
1243 						  p->fence);
1244 		}
1245 	}
1246 }
1247 
1248 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1249 			    union drm_amdgpu_cs *cs)
1250 {
1251 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1252 	struct amdgpu_job *leader = p->gang_leader;
1253 	struct amdgpu_bo_list_entry *e;
1254 	struct drm_gem_object *gobj;
1255 	unsigned long index;
1256 	unsigned int i;
1257 	uint64_t seq;
1258 	int r;
1259 
1260 	for (i = 0; i < p->gang_size; ++i)
1261 		drm_sched_job_arm(&p->jobs[i]->base);
1262 
1263 	for (i = 0; i < p->gang_size; ++i) {
1264 		struct dma_fence *fence;
1265 
1266 		if (p->jobs[i] == leader)
1267 			continue;
1268 
1269 		fence = &p->jobs[i]->base.s_fence->scheduled;
1270 		dma_fence_get(fence);
1271 		r = drm_sched_job_add_dependency(&leader->base, fence);
1272 		if (r) {
1273 			dma_fence_put(fence);
1274 			return r;
1275 		}
1276 	}
1277 
1278 	if (p->gang_size > 1) {
1279 		for (i = 0; i < p->gang_size; ++i)
1280 			amdgpu_job_set_gang_leader(p->jobs[i], leader);
1281 	}
1282 
1283 	/* No memory allocation is allowed while holding the notifier lock.
1284 	 * The lock is held until amdgpu_cs_submit is finished and fence is
1285 	 * added to BOs.
1286 	 */
1287 	mutex_lock(&p->adev->notifier_lock);
1288 
1289 	/* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1290 	 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1291 	 */
1292 	r = 0;
1293 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
1294 		r |= !amdgpu_ttm_tt_get_user_pages_done(e->bo->tbo.ttm,
1295 							e->range);
1296 		e->range = NULL;
1297 	}
1298 	if (r) {
1299 		r = -EAGAIN;
1300 		mutex_unlock(&p->adev->notifier_lock);
1301 		return r;
1302 	}
1303 
1304 	p->fence = dma_fence_get(&leader->base.s_fence->finished);
1305 	drm_exec_for_each_locked_object(&p->exec, index, gobj) {
1306 
1307 		ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo);
1308 
1309 		/* Everybody except for the gang leader uses READ */
1310 		for (i = 0; i < p->gang_size; ++i) {
1311 			if (p->jobs[i] == leader)
1312 				continue;
1313 
1314 			dma_resv_add_fence(gobj->resv,
1315 					   &p->jobs[i]->base.s_fence->finished,
1316 					   DMA_RESV_USAGE_READ);
1317 		}
1318 
1319 		/* The gang leader as remembered as writer */
1320 		dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE);
1321 	}
1322 
1323 	seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx],
1324 				   p->fence);
1325 	amdgpu_cs_post_dependencies(p);
1326 
1327 	if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
1328 	    !p->ctx->preamble_presented) {
1329 		leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
1330 		p->ctx->preamble_presented = true;
1331 	}
1332 
1333 	cs->out.handle = seq;
1334 	leader->uf_sequence = seq;
1335 
1336 	amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->exec.ticket);
1337 	for (i = 0; i < p->gang_size; ++i) {
1338 		amdgpu_job_free_resources(p->jobs[i]);
1339 		trace_amdgpu_cs_ioctl(p->jobs[i]);
1340 		drm_sched_entity_push_job(&p->jobs[i]->base);
1341 		p->jobs[i] = NULL;
1342 	}
1343 
1344 	amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
1345 
1346 	mutex_unlock(&p->adev->notifier_lock);
1347 	mutex_unlock(&p->bo_list->bo_list_mutex);
1348 	return 0;
1349 }
1350 
1351 /* Cleanup the parser structure */
1352 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)
1353 {
1354 	unsigned int i;
1355 
1356 	amdgpu_sync_free(&parser->sync);
1357 	drm_exec_fini(&parser->exec);
1358 
1359 	for (i = 0; i < parser->num_post_deps; i++) {
1360 		drm_syncobj_put(parser->post_deps[i].syncobj);
1361 		kfree(parser->post_deps[i].chain);
1362 	}
1363 	kfree(parser->post_deps);
1364 
1365 	dma_fence_put(parser->fence);
1366 
1367 	if (parser->ctx)
1368 		amdgpu_ctx_put(parser->ctx);
1369 	if (parser->bo_list)
1370 		amdgpu_bo_list_put(parser->bo_list);
1371 
1372 	for (i = 0; i < parser->nchunks; i++)
1373 		kvfree(parser->chunks[i].kdata);
1374 	kvfree(parser->chunks);
1375 	for (i = 0; i < parser->gang_size; ++i) {
1376 		if (parser->jobs[i])
1377 			amdgpu_job_free(parser->jobs[i]);
1378 	}
1379 	amdgpu_bo_unref(&parser->uf_bo);
1380 }
1381 
1382 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1383 {
1384 	struct amdgpu_device *adev = drm_to_adev(dev);
1385 	struct amdgpu_cs_parser parser;
1386 	int r;
1387 
1388 	if (amdgpu_ras_intr_triggered())
1389 		return -EHWPOISON;
1390 
1391 	if (!adev->accel_working)
1392 		return -EBUSY;
1393 
1394 	r = amdgpu_cs_parser_init(&parser, adev, filp, data);
1395 	if (r) {
1396 		if (printk_ratelimit())
1397 			DRM_ERROR("Failed to initialize parser %d!\n", r);
1398 		return r;
1399 	}
1400 
1401 	r = amdgpu_cs_pass1(&parser, data);
1402 	if (r)
1403 		goto error_fini;
1404 
1405 	r = amdgpu_cs_pass2(&parser);
1406 	if (r)
1407 		goto error_fini;
1408 
1409 	r = amdgpu_cs_parser_bos(&parser, data);
1410 	if (r) {
1411 		if (r == -ENOMEM)
1412 			DRM_ERROR("Not enough memory for command submission!\n");
1413 		else if (r != -ERESTARTSYS && r != -EAGAIN)
1414 			DRM_DEBUG("Failed to process the buffer list %d!\n", r);
1415 		goto error_fini;
1416 	}
1417 
1418 	r = amdgpu_cs_patch_jobs(&parser);
1419 	if (r)
1420 		goto error_backoff;
1421 
1422 	r = amdgpu_cs_vm_handling(&parser);
1423 	if (r)
1424 		goto error_backoff;
1425 
1426 	r = amdgpu_cs_sync_rings(&parser);
1427 	if (r)
1428 		goto error_backoff;
1429 
1430 	trace_amdgpu_cs_ibs(&parser);
1431 
1432 	r = amdgpu_cs_submit(&parser, data);
1433 	if (r)
1434 		goto error_backoff;
1435 
1436 	amdgpu_cs_parser_fini(&parser);
1437 	return 0;
1438 
1439 error_backoff:
1440 	mutex_unlock(&parser.bo_list->bo_list_mutex);
1441 
1442 error_fini:
1443 	amdgpu_cs_parser_fini(&parser);
1444 	return r;
1445 }
1446 
1447 /**
1448  * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1449  *
1450  * @dev: drm device
1451  * @data: data from userspace
1452  * @filp: file private
1453  *
1454  * Wait for the command submission identified by handle to finish.
1455  */
1456 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
1457 			 struct drm_file *filp)
1458 {
1459 	union drm_amdgpu_wait_cs *wait = data;
1460 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1461 	struct drm_sched_entity *entity;
1462 	struct amdgpu_ctx *ctx;
1463 	struct dma_fence *fence;
1464 	long r;
1465 
1466 	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
1467 	if (ctx == NULL)
1468 		return -EINVAL;
1469 
1470 	r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,
1471 				  wait->in.ring, &entity);
1472 	if (r) {
1473 		amdgpu_ctx_put(ctx);
1474 		return r;
1475 	}
1476 
1477 	fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);
1478 	if (IS_ERR(fence))
1479 		r = PTR_ERR(fence);
1480 	else if (fence) {
1481 		r = dma_fence_wait_timeout(fence, true, timeout);
1482 		if (r > 0 && fence->error)
1483 			r = fence->error;
1484 		dma_fence_put(fence);
1485 	} else
1486 		r = 1;
1487 
1488 	amdgpu_ctx_put(ctx);
1489 	if (r < 0)
1490 		return r;
1491 
1492 	memset(wait, 0, sizeof(*wait));
1493 	wait->out.status = (r == 0);
1494 
1495 	return 0;
1496 }
1497 
1498 /**
1499  * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1500  *
1501  * @adev: amdgpu device
1502  * @filp: file private
1503  * @user: drm_amdgpu_fence copied from user space
1504  */
1505 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1506 					     struct drm_file *filp,
1507 					     struct drm_amdgpu_fence *user)
1508 {
1509 	struct drm_sched_entity *entity;
1510 	struct amdgpu_ctx *ctx;
1511 	struct dma_fence *fence;
1512 	int r;
1513 
1514 	ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
1515 	if (ctx == NULL)
1516 		return ERR_PTR(-EINVAL);
1517 
1518 	r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,
1519 				  user->ring, &entity);
1520 	if (r) {
1521 		amdgpu_ctx_put(ctx);
1522 		return ERR_PTR(r);
1523 	}
1524 
1525 	fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);
1526 	amdgpu_ctx_put(ctx);
1527 
1528 	return fence;
1529 }
1530 
1531 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1532 				    struct drm_file *filp)
1533 {
1534 	struct amdgpu_device *adev = drm_to_adev(dev);
1535 	union drm_amdgpu_fence_to_handle *info = data;
1536 	struct dma_fence *fence;
1537 	struct drm_syncobj *syncobj;
1538 	struct sync_file *sync_file;
1539 	int fd, r;
1540 
1541 	fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1542 	if (IS_ERR(fence))
1543 		return PTR_ERR(fence);
1544 
1545 	if (!fence)
1546 		fence = dma_fence_get_stub();
1547 
1548 	switch (info->in.what) {
1549 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1550 		r = drm_syncobj_create(&syncobj, 0, fence);
1551 		dma_fence_put(fence);
1552 		if (r)
1553 			return r;
1554 		r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1555 		drm_syncobj_put(syncobj);
1556 		return r;
1557 
1558 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1559 		r = drm_syncobj_create(&syncobj, 0, fence);
1560 		dma_fence_put(fence);
1561 		if (r)
1562 			return r;
1563 		r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);
1564 		drm_syncobj_put(syncobj);
1565 		return r;
1566 
1567 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1568 		fd = get_unused_fd_flags(O_CLOEXEC);
1569 		if (fd < 0) {
1570 			dma_fence_put(fence);
1571 			return fd;
1572 		}
1573 
1574 		sync_file = sync_file_create(fence);
1575 		dma_fence_put(fence);
1576 		if (!sync_file) {
1577 			put_unused_fd(fd);
1578 			return -ENOMEM;
1579 		}
1580 
1581 		fd_install(fd, sync_file->file);
1582 		info->out.handle = fd;
1583 		return 0;
1584 
1585 	default:
1586 		dma_fence_put(fence);
1587 		return -EINVAL;
1588 	}
1589 }
1590 
1591 /**
1592  * amdgpu_cs_wait_all_fences - wait on all fences to signal
1593  *
1594  * @adev: amdgpu device
1595  * @filp: file private
1596  * @wait: wait parameters
1597  * @fences: array of drm_amdgpu_fence
1598  */
1599 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
1600 				     struct drm_file *filp,
1601 				     union drm_amdgpu_wait_fences *wait,
1602 				     struct drm_amdgpu_fence *fences)
1603 {
1604 	uint32_t fence_count = wait->in.fence_count;
1605 	unsigned int i;
1606 	long r = 1;
1607 
1608 	for (i = 0; i < fence_count; i++) {
1609 		struct dma_fence *fence;
1610 		unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1611 
1612 		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1613 		if (IS_ERR(fence))
1614 			return PTR_ERR(fence);
1615 		else if (!fence)
1616 			continue;
1617 
1618 		r = dma_fence_wait_timeout(fence, true, timeout);
1619 		if (r > 0 && fence->error)
1620 			r = fence->error;
1621 
1622 		dma_fence_put(fence);
1623 		if (r < 0)
1624 			return r;
1625 
1626 		if (r == 0)
1627 			break;
1628 	}
1629 
1630 	memset(wait, 0, sizeof(*wait));
1631 	wait->out.status = (r > 0);
1632 
1633 	return 0;
1634 }
1635 
1636 /**
1637  * amdgpu_cs_wait_any_fence - wait on any fence to signal
1638  *
1639  * @adev: amdgpu device
1640  * @filp: file private
1641  * @wait: wait parameters
1642  * @fences: array of drm_amdgpu_fence
1643  */
1644 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
1645 				    struct drm_file *filp,
1646 				    union drm_amdgpu_wait_fences *wait,
1647 				    struct drm_amdgpu_fence *fences)
1648 {
1649 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1650 	uint32_t fence_count = wait->in.fence_count;
1651 	uint32_t first = ~0;
1652 	struct dma_fence **array;
1653 	unsigned int i;
1654 	long r;
1655 
1656 	/* Prepare the fence array */
1657 	array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
1658 
1659 	if (array == NULL)
1660 		return -ENOMEM;
1661 
1662 	for (i = 0; i < fence_count; i++) {
1663 		struct dma_fence *fence;
1664 
1665 		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1666 		if (IS_ERR(fence)) {
1667 			r = PTR_ERR(fence);
1668 			goto err_free_fence_array;
1669 		} else if (fence) {
1670 			array[i] = fence;
1671 		} else { /* NULL, the fence has been already signaled */
1672 			r = 1;
1673 			first = i;
1674 			goto out;
1675 		}
1676 	}
1677 
1678 	r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
1679 				       &first);
1680 	if (r < 0)
1681 		goto err_free_fence_array;
1682 
1683 out:
1684 	memset(wait, 0, sizeof(*wait));
1685 	wait->out.status = (r > 0);
1686 	wait->out.first_signaled = first;
1687 
1688 	if (first < fence_count && array[first])
1689 		r = array[first]->error;
1690 	else
1691 		r = 0;
1692 
1693 err_free_fence_array:
1694 	for (i = 0; i < fence_count; i++)
1695 		dma_fence_put(array[i]);
1696 	kfree(array);
1697 
1698 	return r;
1699 }
1700 
1701 /**
1702  * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1703  *
1704  * @dev: drm device
1705  * @data: data from userspace
1706  * @filp: file private
1707  */
1708 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1709 				struct drm_file *filp)
1710 {
1711 	struct amdgpu_device *adev = drm_to_adev(dev);
1712 	union drm_amdgpu_wait_fences *wait = data;
1713 	uint32_t fence_count = wait->in.fence_count;
1714 	struct drm_amdgpu_fence *fences_user;
1715 	struct drm_amdgpu_fence *fences;
1716 	int r;
1717 
1718 	/* Get the fences from userspace */
1719 	fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
1720 			GFP_KERNEL);
1721 	if (fences == NULL)
1722 		return -ENOMEM;
1723 
1724 	fences_user = u64_to_user_ptr(wait->in.fences);
1725 	if (copy_from_user(fences, fences_user,
1726 		sizeof(struct drm_amdgpu_fence) * fence_count)) {
1727 		r = -EFAULT;
1728 		goto err_free_fences;
1729 	}
1730 
1731 	if (wait->in.wait_all)
1732 		r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
1733 	else
1734 		r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
1735 
1736 err_free_fences:
1737 	kfree(fences);
1738 
1739 	return r;
1740 }
1741 
1742 /**
1743  * amdgpu_cs_find_mapping - find bo_va for VM address
1744  *
1745  * @parser: command submission parser context
1746  * @addr: VM address
1747  * @bo: resulting BO of the mapping found
1748  * @map: Placeholder to return found BO mapping
1749  *
1750  * Search the buffer objects in the command submission context for a certain
1751  * virtual memory address. Returns allocation structure when found, NULL
1752  * otherwise.
1753  */
1754 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1755 			   uint64_t addr, struct amdgpu_bo **bo,
1756 			   struct amdgpu_bo_va_mapping **map)
1757 {
1758 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1759 	struct ttm_operation_ctx ctx = { false, false };
1760 	struct amdgpu_vm *vm = &fpriv->vm;
1761 	struct amdgpu_bo_va_mapping *mapping;
1762 	int r;
1763 
1764 	addr /= AMDGPU_GPU_PAGE_SIZE;
1765 
1766 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
1767 	if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1768 		return -EINVAL;
1769 
1770 	*bo = mapping->bo_va->base.bo;
1771 	*map = mapping;
1772 
1773 	/* Double check that the BO is reserved by this CS */
1774 	if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->exec.ticket)
1775 		return -EINVAL;
1776 
1777 	if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
1778 		(*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1779 		amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1780 		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1781 		if (r)
1782 			return r;
1783 	}
1784 
1785 	return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1786 }
1787