1 /*
2  * Copyright 2008 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22  * DEALINGS IN THE SOFTWARE.
23  *
24  * Authors:
25  *    Jerome Glisse <glisse@freedesktop.org>
26  */
27 
28 #include <linux/file.h>
29 #include <linux/pagemap.h>
30 #include <linux/sync_file.h>
31 #include <linux/dma-buf.h>
32 
33 #include <drm/amdgpu_drm.h>
34 #include <drm/drm_syncobj.h>
35 #include "amdgpu_cs.h"
36 #include "amdgpu.h"
37 #include "amdgpu_trace.h"
38 #include "amdgpu_gmc.h"
39 #include "amdgpu_gem.h"
40 #include "amdgpu_ras.h"
41 
42 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,
43 				 struct amdgpu_device *adev,
44 				 struct drm_file *filp,
45 				 union drm_amdgpu_cs *cs)
46 {
47 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
48 
49 	if (cs->in.num_chunks == 0)
50 		return -EINVAL;
51 
52 	memset(p, 0, sizeof(*p));
53 	p->adev = adev;
54 	p->filp = filp;
55 
56 	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
57 	if (!p->ctx)
58 		return -EINVAL;
59 
60 	if (atomic_read(&p->ctx->guilty)) {
61 		amdgpu_ctx_put(p->ctx);
62 		return -ECANCELED;
63 	}
64 	return 0;
65 }
66 
67 static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
68 			     struct drm_amdgpu_cs_chunk_ib *chunk_ib)
69 {
70 	struct drm_sched_entity *entity;
71 	unsigned int i;
72 	int r;
73 
74 	r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,
75 				  chunk_ib->ip_instance,
76 				  chunk_ib->ring, &entity);
77 	if (r)
78 		return r;
79 
80 	/*
81 	 * Abort if there is no run queue associated with this entity.
82 	 * Possibly because of disabled HW IP.
83 	 */
84 	if (entity->rq == NULL)
85 		return -EINVAL;
86 
87 	/* Check if we can add this IB to some existing job */
88 	for (i = 0; i < p->gang_size; ++i)
89 		if (p->entities[i] == entity)
90 			return i;
91 
92 	/* If not increase the gang size if possible */
93 	if (i == AMDGPU_CS_GANG_SIZE)
94 		return -EINVAL;
95 
96 	p->entities[i] = entity;
97 	p->gang_size = i + 1;
98 	return i;
99 }
100 
101 static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
102 			   struct drm_amdgpu_cs_chunk_ib *chunk_ib,
103 			   unsigned int *num_ibs)
104 {
105 	int r;
106 
107 	r = amdgpu_cs_job_idx(p, chunk_ib);
108 	if (r < 0)
109 		return r;
110 
111 	++(num_ibs[r]);
112 	return 0;
113 }
114 
115 static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,
116 				   struct drm_amdgpu_cs_chunk_fence *data,
117 				   uint32_t *offset)
118 {
119 	struct drm_gem_object *gobj;
120 	struct amdgpu_bo *bo;
121 	unsigned long size;
122 	int r;
123 
124 	gobj = drm_gem_object_lookup(p->filp, data->handle);
125 	if (gobj == NULL)
126 		return -EINVAL;
127 
128 	bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
129 	p->uf_entry.priority = 0;
130 	p->uf_entry.tv.bo = &bo->tbo;
131 	/* One for TTM and two for the CS job */
132 	p->uf_entry.tv.num_shared = 3;
133 
134 	drm_gem_object_put(gobj);
135 
136 	size = amdgpu_bo_size(bo);
137 	if (size != PAGE_SIZE || (data->offset + 8) > size) {
138 		r = -EINVAL;
139 		goto error_unref;
140 	}
141 
142 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
143 		r = -EINVAL;
144 		goto error_unref;
145 	}
146 
147 	*offset = data->offset;
148 
149 	return 0;
150 
151 error_unref:
152 	amdgpu_bo_unref(&bo);
153 	return r;
154 }
155 
156 static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
157 				   struct drm_amdgpu_bo_list_in *data)
158 {
159 	struct drm_amdgpu_bo_list_entry *info;
160 	int r;
161 
162 	r = amdgpu_bo_create_list_entry_array(data, &info);
163 	if (r)
164 		return r;
165 
166 	r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
167 				  &p->bo_list);
168 	if (r)
169 		goto error_free;
170 
171 	kvfree(info);
172 	return 0;
173 
174 error_free:
175 	kvfree(info);
176 
177 	return r;
178 }
179 
180 /* Copy the data from userspace and go over it the first time */
181 static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
182 			   union drm_amdgpu_cs *cs)
183 {
184 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
185 	unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };
186 	struct amdgpu_vm *vm = &fpriv->vm;
187 	uint64_t *chunk_array_user;
188 	uint64_t *chunk_array;
189 	uint32_t uf_offset = 0;
190 	unsigned int size;
191 	int ret;
192 	int i;
193 
194 	chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t),
195 				     GFP_KERNEL);
196 	if (!chunk_array)
197 		return -ENOMEM;
198 
199 	/* get chunks */
200 	chunk_array_user = u64_to_user_ptr(cs->in.chunks);
201 	if (copy_from_user(chunk_array, chunk_array_user,
202 			   sizeof(uint64_t)*cs->in.num_chunks)) {
203 		ret = -EFAULT;
204 		goto free_chunk;
205 	}
206 
207 	p->nchunks = cs->in.num_chunks;
208 	p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
209 			    GFP_KERNEL);
210 	if (!p->chunks) {
211 		ret = -ENOMEM;
212 		goto free_chunk;
213 	}
214 
215 	for (i = 0; i < p->nchunks; i++) {
216 		struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
217 		struct drm_amdgpu_cs_chunk user_chunk;
218 		uint32_t __user *cdata;
219 
220 		chunk_ptr = u64_to_user_ptr(chunk_array[i]);
221 		if (copy_from_user(&user_chunk, chunk_ptr,
222 				       sizeof(struct drm_amdgpu_cs_chunk))) {
223 			ret = -EFAULT;
224 			i--;
225 			goto free_partial_kdata;
226 		}
227 		p->chunks[i].chunk_id = user_chunk.chunk_id;
228 		p->chunks[i].length_dw = user_chunk.length_dw;
229 
230 		size = p->chunks[i].length_dw;
231 		cdata = u64_to_user_ptr(user_chunk.chunk_data);
232 
233 		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t),
234 						    GFP_KERNEL);
235 		if (p->chunks[i].kdata == NULL) {
236 			ret = -ENOMEM;
237 			i--;
238 			goto free_partial_kdata;
239 		}
240 		size *= sizeof(uint32_t);
241 		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
242 			ret = -EFAULT;
243 			goto free_partial_kdata;
244 		}
245 
246 		/* Assume the worst on the following checks */
247 		ret = -EINVAL;
248 		switch (p->chunks[i].chunk_id) {
249 		case AMDGPU_CHUNK_ID_IB:
250 			if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))
251 				goto free_partial_kdata;
252 
253 			ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);
254 			if (ret)
255 				goto free_partial_kdata;
256 			break;
257 
258 		case AMDGPU_CHUNK_ID_FENCE:
259 			if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))
260 				goto free_partial_kdata;
261 
262 			ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,
263 						      &uf_offset);
264 			if (ret)
265 				goto free_partial_kdata;
266 			break;
267 
268 		case AMDGPU_CHUNK_ID_BO_HANDLES:
269 			if (size < sizeof(struct drm_amdgpu_bo_list_in))
270 				goto free_partial_kdata;
271 
272 			ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);
273 			if (ret)
274 				goto free_partial_kdata;
275 			break;
276 
277 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
278 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
279 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
280 		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
281 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
282 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
283 			break;
284 
285 		default:
286 			goto free_partial_kdata;
287 		}
288 	}
289 
290 	if (!p->gang_size)
291 		return -EINVAL;
292 
293 	for (i = 0; i < p->gang_size; ++i) {
294 		ret = amdgpu_job_alloc(p->adev, num_ibs[i], &p->jobs[i], vm);
295 		if (ret)
296 			goto free_all_kdata;
297 
298 		ret = drm_sched_job_init(&p->jobs[i]->base, p->entities[i],
299 					 &fpriv->vm);
300 		if (ret)
301 			goto free_all_kdata;
302 	}
303 	p->gang_leader = p->jobs[p->gang_size - 1];
304 
305 	if (p->ctx->vram_lost_counter != p->gang_leader->vram_lost_counter) {
306 		ret = -ECANCELED;
307 		goto free_all_kdata;
308 	}
309 
310 	if (p->uf_entry.tv.bo)
311 		p->gang_leader->uf_addr = uf_offset;
312 	kvfree(chunk_array);
313 
314 	/* Use this opportunity to fill in task info for the vm */
315 	amdgpu_vm_set_task_info(vm);
316 
317 	return 0;
318 
319 free_all_kdata:
320 	i = p->nchunks - 1;
321 free_partial_kdata:
322 	for (; i >= 0; i--)
323 		kvfree(p->chunks[i].kdata);
324 	kvfree(p->chunks);
325 	p->chunks = NULL;
326 	p->nchunks = 0;
327 free_chunk:
328 	kvfree(chunk_array);
329 
330 	return ret;
331 }
332 
333 static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
334 			   struct amdgpu_cs_chunk *chunk,
335 			   unsigned int *ce_preempt,
336 			   unsigned int *de_preempt)
337 {
338 	struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;
339 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
340 	struct amdgpu_vm *vm = &fpriv->vm;
341 	struct amdgpu_ring *ring;
342 	struct amdgpu_job *job;
343 	struct amdgpu_ib *ib;
344 	int r;
345 
346 	r = amdgpu_cs_job_idx(p, chunk_ib);
347 	if (r < 0)
348 		return r;
349 
350 	job = p->jobs[r];
351 	ring = amdgpu_job_ring(job);
352 	ib = &job->ibs[job->num_ibs++];
353 
354 	/* MM engine doesn't support user fences */
355 	if (p->uf_entry.tv.bo && ring->funcs->no_user_fence)
356 		return -EINVAL;
357 
358 	if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
359 	    chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
360 		if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
361 			(*ce_preempt)++;
362 		else
363 			(*de_preempt)++;
364 
365 		/* Each GFX command submit allows only 1 IB max
366 		 * preemptible for CE & DE */
367 		if (*ce_preempt > 1 || *de_preempt > 1)
368 			return -EINVAL;
369 	}
370 
371 	if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
372 		job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
373 
374 	r =  amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?
375 			   chunk_ib->ib_bytes : 0,
376 			   AMDGPU_IB_POOL_DELAYED, ib);
377 	if (r) {
378 		DRM_ERROR("Failed to get ib !\n");
379 		return r;
380 	}
381 
382 	ib->gpu_addr = chunk_ib->va_start;
383 	ib->length_dw = chunk_ib->ib_bytes / 4;
384 	ib->flags = chunk_ib->flags;
385 	return 0;
386 }
387 
388 static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
389 				     struct amdgpu_cs_chunk *chunk)
390 {
391 	struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;
392 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
393 	unsigned num_deps;
394 	int i, r;
395 
396 	num_deps = chunk->length_dw * 4 /
397 		sizeof(struct drm_amdgpu_cs_chunk_dep);
398 
399 	for (i = 0; i < num_deps; ++i) {
400 		struct amdgpu_ctx *ctx;
401 		struct drm_sched_entity *entity;
402 		struct dma_fence *fence;
403 
404 		ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
405 		if (ctx == NULL)
406 			return -EINVAL;
407 
408 		r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
409 					  deps[i].ip_instance,
410 					  deps[i].ring, &entity);
411 		if (r) {
412 			amdgpu_ctx_put(ctx);
413 			return r;
414 		}
415 
416 		fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
417 		amdgpu_ctx_put(ctx);
418 
419 		if (IS_ERR(fence))
420 			return PTR_ERR(fence);
421 		else if (!fence)
422 			continue;
423 
424 		if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
425 			struct drm_sched_fence *s_fence;
426 			struct dma_fence *old = fence;
427 
428 			s_fence = to_drm_sched_fence(fence);
429 			fence = dma_fence_get(&s_fence->scheduled);
430 			dma_fence_put(old);
431 		}
432 
433 		r = amdgpu_sync_fence(&p->gang_leader->sync, fence);
434 		dma_fence_put(fence);
435 		if (r)
436 			return r;
437 	}
438 	return 0;
439 }
440 
441 static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
442 					 uint32_t handle, u64 point,
443 					 u64 flags)
444 {
445 	struct dma_fence *fence;
446 	int r;
447 
448 	r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
449 	if (r) {
450 		DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
451 			  handle, point, r);
452 		return r;
453 	}
454 
455 	r = amdgpu_sync_fence(&p->gang_leader->sync, fence);
456 	dma_fence_put(fence);
457 
458 	return r;
459 }
460 
461 static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,
462 				   struct amdgpu_cs_chunk *chunk)
463 {
464 	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
465 	unsigned num_deps;
466 	int i, r;
467 
468 	num_deps = chunk->length_dw * 4 /
469 		sizeof(struct drm_amdgpu_cs_chunk_sem);
470 	for (i = 0; i < num_deps; ++i) {
471 		r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);
472 		if (r)
473 			return r;
474 	}
475 
476 	return 0;
477 }
478 
479 static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,
480 					      struct amdgpu_cs_chunk *chunk)
481 {
482 	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
483 	unsigned num_deps;
484 	int i, r;
485 
486 	num_deps = chunk->length_dw * 4 /
487 		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
488 	for (i = 0; i < num_deps; ++i) {
489 		r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,
490 						  syncobj_deps[i].point,
491 						  syncobj_deps[i].flags);
492 		if (r)
493 			return r;
494 	}
495 
496 	return 0;
497 }
498 
499 static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,
500 				    struct amdgpu_cs_chunk *chunk)
501 {
502 	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
503 	unsigned num_deps;
504 	int i;
505 
506 	num_deps = chunk->length_dw * 4 /
507 		sizeof(struct drm_amdgpu_cs_chunk_sem);
508 
509 	if (p->post_deps)
510 		return -EINVAL;
511 
512 	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
513 				     GFP_KERNEL);
514 	p->num_post_deps = 0;
515 
516 	if (!p->post_deps)
517 		return -ENOMEM;
518 
519 
520 	for (i = 0; i < num_deps; ++i) {
521 		p->post_deps[i].syncobj =
522 			drm_syncobj_find(p->filp, deps[i].handle);
523 		if (!p->post_deps[i].syncobj)
524 			return -EINVAL;
525 		p->post_deps[i].chain = NULL;
526 		p->post_deps[i].point = 0;
527 		p->num_post_deps++;
528 	}
529 
530 	return 0;
531 }
532 
533 static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,
534 						struct amdgpu_cs_chunk *chunk)
535 {
536 	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
537 	unsigned num_deps;
538 	int i;
539 
540 	num_deps = chunk->length_dw * 4 /
541 		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
542 
543 	if (p->post_deps)
544 		return -EINVAL;
545 
546 	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
547 				     GFP_KERNEL);
548 	p->num_post_deps = 0;
549 
550 	if (!p->post_deps)
551 		return -ENOMEM;
552 
553 	for (i = 0; i < num_deps; ++i) {
554 		struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
555 
556 		dep->chain = NULL;
557 		if (syncobj_deps[i].point) {
558 			dep->chain = dma_fence_chain_alloc();
559 			if (!dep->chain)
560 				return -ENOMEM;
561 		}
562 
563 		dep->syncobj = drm_syncobj_find(p->filp,
564 						syncobj_deps[i].handle);
565 		if (!dep->syncobj) {
566 			dma_fence_chain_free(dep->chain);
567 			return -EINVAL;
568 		}
569 		dep->point = syncobj_deps[i].point;
570 		p->num_post_deps++;
571 	}
572 
573 	return 0;
574 }
575 
576 static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)
577 {
578 	unsigned int ce_preempt = 0, de_preempt = 0;
579 	int i, r;
580 
581 	for (i = 0; i < p->nchunks; ++i) {
582 		struct amdgpu_cs_chunk *chunk;
583 
584 		chunk = &p->chunks[i];
585 
586 		switch (chunk->chunk_id) {
587 		case AMDGPU_CHUNK_ID_IB:
588 			r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);
589 			if (r)
590 				return r;
591 			break;
592 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
593 		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
594 			r = amdgpu_cs_p2_dependencies(p, chunk);
595 			if (r)
596 				return r;
597 			break;
598 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
599 			r = amdgpu_cs_p2_syncobj_in(p, chunk);
600 			if (r)
601 				return r;
602 			break;
603 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
604 			r = amdgpu_cs_p2_syncobj_out(p, chunk);
605 			if (r)
606 				return r;
607 			break;
608 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
609 			r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);
610 			if (r)
611 				return r;
612 			break;
613 		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
614 			r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);
615 			if (r)
616 				return r;
617 			break;
618 		}
619 	}
620 
621 	return 0;
622 }
623 
624 /* Convert microseconds to bytes. */
625 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
626 {
627 	if (us <= 0 || !adev->mm_stats.log2_max_MBps)
628 		return 0;
629 
630 	/* Since accum_us is incremented by a million per second, just
631 	 * multiply it by the number of MB/s to get the number of bytes.
632 	 */
633 	return us << adev->mm_stats.log2_max_MBps;
634 }
635 
636 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
637 {
638 	if (!adev->mm_stats.log2_max_MBps)
639 		return 0;
640 
641 	return bytes >> adev->mm_stats.log2_max_MBps;
642 }
643 
644 /* Returns how many bytes TTM can move right now. If no bytes can be moved,
645  * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
646  * which means it can go over the threshold once. If that happens, the driver
647  * will be in debt and no other buffer migrations can be done until that debt
648  * is repaid.
649  *
650  * This approach allows moving a buffer of any size (it's important to allow
651  * that).
652  *
653  * The currency is simply time in microseconds and it increases as the clock
654  * ticks. The accumulated microseconds (us) are converted to bytes and
655  * returned.
656  */
657 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
658 					      u64 *max_bytes,
659 					      u64 *max_vis_bytes)
660 {
661 	s64 time_us, increment_us;
662 	u64 free_vram, total_vram, used_vram;
663 	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
664 	 * throttling.
665 	 *
666 	 * It means that in order to get full max MBps, at least 5 IBs per
667 	 * second must be submitted and not more than 200ms apart from each
668 	 * other.
669 	 */
670 	const s64 us_upper_bound = 200000;
671 
672 	if (!adev->mm_stats.log2_max_MBps) {
673 		*max_bytes = 0;
674 		*max_vis_bytes = 0;
675 		return;
676 	}
677 
678 	total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
679 	used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager);
680 	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
681 
682 	spin_lock(&adev->mm_stats.lock);
683 
684 	/* Increase the amount of accumulated us. */
685 	time_us = ktime_to_us(ktime_get());
686 	increment_us = time_us - adev->mm_stats.last_update_us;
687 	adev->mm_stats.last_update_us = time_us;
688 	adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
689 				      us_upper_bound);
690 
691 	/* This prevents the short period of low performance when the VRAM
692 	 * usage is low and the driver is in debt or doesn't have enough
693 	 * accumulated us to fill VRAM quickly.
694 	 *
695 	 * The situation can occur in these cases:
696 	 * - a lot of VRAM is freed by userspace
697 	 * - the presence of a big buffer causes a lot of evictions
698 	 *   (solution: split buffers into smaller ones)
699 	 *
700 	 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
701 	 * accum_us to a positive number.
702 	 */
703 	if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
704 		s64 min_us;
705 
706 		/* Be more aggressive on dGPUs. Try to fill a portion of free
707 		 * VRAM now.
708 		 */
709 		if (!(adev->flags & AMD_IS_APU))
710 			min_us = bytes_to_us(adev, free_vram / 4);
711 		else
712 			min_us = 0; /* Reset accum_us on APUs. */
713 
714 		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
715 	}
716 
717 	/* This is set to 0 if the driver is in debt to disallow (optional)
718 	 * buffer moves.
719 	 */
720 	*max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
721 
722 	/* Do the same for visible VRAM if half of it is free */
723 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
724 		u64 total_vis_vram = adev->gmc.visible_vram_size;
725 		u64 used_vis_vram =
726 		  amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
727 
728 		if (used_vis_vram < total_vis_vram) {
729 			u64 free_vis_vram = total_vis_vram - used_vis_vram;
730 			adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
731 							  increment_us, us_upper_bound);
732 
733 			if (free_vis_vram >= total_vis_vram / 2)
734 				adev->mm_stats.accum_us_vis =
735 					max(bytes_to_us(adev, free_vis_vram / 2),
736 					    adev->mm_stats.accum_us_vis);
737 		}
738 
739 		*max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
740 	} else {
741 		*max_vis_bytes = 0;
742 	}
743 
744 	spin_unlock(&adev->mm_stats.lock);
745 }
746 
747 /* Report how many bytes have really been moved for the last command
748  * submission. This can result in a debt that can stop buffer migrations
749  * temporarily.
750  */
751 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
752 				  u64 num_vis_bytes)
753 {
754 	spin_lock(&adev->mm_stats.lock);
755 	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
756 	adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
757 	spin_unlock(&adev->mm_stats.lock);
758 }
759 
760 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)
761 {
762 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
763 	struct amdgpu_cs_parser *p = param;
764 	struct ttm_operation_ctx ctx = {
765 		.interruptible = true,
766 		.no_wait_gpu = false,
767 		.resv = bo->tbo.base.resv
768 	};
769 	uint32_t domain;
770 	int r;
771 
772 	if (bo->tbo.pin_count)
773 		return 0;
774 
775 	/* Don't move this buffer if we have depleted our allowance
776 	 * to move it. Don't move anything if the threshold is zero.
777 	 */
778 	if (p->bytes_moved < p->bytes_moved_threshold &&
779 	    (!bo->tbo.base.dma_buf ||
780 	    list_empty(&bo->tbo.base.dma_buf->attachments))) {
781 		if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
782 		    (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
783 			/* And don't move a CPU_ACCESS_REQUIRED BO to limited
784 			 * visible VRAM if we've depleted our allowance to do
785 			 * that.
786 			 */
787 			if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
788 				domain = bo->preferred_domains;
789 			else
790 				domain = bo->allowed_domains;
791 		} else {
792 			domain = bo->preferred_domains;
793 		}
794 	} else {
795 		domain = bo->allowed_domains;
796 	}
797 
798 retry:
799 	amdgpu_bo_placement_from_domain(bo, domain);
800 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
801 
802 	p->bytes_moved += ctx.bytes_moved;
803 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
804 	    amdgpu_bo_in_cpu_visible_vram(bo))
805 		p->bytes_moved_vis += ctx.bytes_moved;
806 
807 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
808 		domain = bo->allowed_domains;
809 		goto retry;
810 	}
811 
812 	return r;
813 }
814 
815 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
816 			    struct list_head *validated)
817 {
818 	struct ttm_operation_ctx ctx = { true, false };
819 	struct amdgpu_bo_list_entry *lobj;
820 	int r;
821 
822 	list_for_each_entry(lobj, validated, tv.head) {
823 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo);
824 		struct mm_struct *usermm;
825 
826 		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
827 		if (usermm && usermm != current->mm)
828 			return -EPERM;
829 
830 		if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) &&
831 		    lobj->user_invalidated && lobj->user_pages) {
832 			amdgpu_bo_placement_from_domain(bo,
833 							AMDGPU_GEM_DOMAIN_CPU);
834 			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
835 			if (r)
836 				return r;
837 
838 			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
839 						     lobj->user_pages);
840 		}
841 
842 		r = amdgpu_cs_bo_validate(p, bo);
843 		if (r)
844 			return r;
845 
846 		kvfree(lobj->user_pages);
847 		lobj->user_pages = NULL;
848 	}
849 	return 0;
850 }
851 
852 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
853 				union drm_amdgpu_cs *cs)
854 {
855 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
856 	struct amdgpu_vm *vm = &fpriv->vm;
857 	struct amdgpu_bo_list_entry *e;
858 	struct list_head duplicates;
859 	unsigned int i;
860 	int r;
861 
862 	INIT_LIST_HEAD(&p->validated);
863 
864 	/* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
865 	if (cs->in.bo_list_handle) {
866 		if (p->bo_list)
867 			return -EINVAL;
868 
869 		r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
870 				       &p->bo_list);
871 		if (r)
872 			return r;
873 	} else if (!p->bo_list) {
874 		/* Create a empty bo_list when no handle is provided */
875 		r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
876 					  &p->bo_list);
877 		if (r)
878 			return r;
879 	}
880 
881 	mutex_lock(&p->bo_list->bo_list_mutex);
882 
883 	/* One for TTM and one for the CS job */
884 	amdgpu_bo_list_for_each_entry(e, p->bo_list)
885 		e->tv.num_shared = 2;
886 
887 	amdgpu_bo_list_get_list(p->bo_list, &p->validated);
888 
889 	INIT_LIST_HEAD(&duplicates);
890 	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
891 
892 	if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent)
893 		list_add(&p->uf_entry.tv.head, &p->validated);
894 
895 	/* Get userptr backing pages. If pages are updated after registered
896 	 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
897 	 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
898 	 */
899 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
900 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
901 		bool userpage_invalidated = false;
902 		int i;
903 
904 		e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
905 					sizeof(struct page *),
906 					GFP_KERNEL | __GFP_ZERO);
907 		if (!e->user_pages) {
908 			DRM_ERROR("kvmalloc_array failure\n");
909 			r = -ENOMEM;
910 			goto out_free_user_pages;
911 		}
912 
913 		r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages);
914 		if (r) {
915 			kvfree(e->user_pages);
916 			e->user_pages = NULL;
917 			goto out_free_user_pages;
918 		}
919 
920 		for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
921 			if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
922 				userpage_invalidated = true;
923 				break;
924 			}
925 		}
926 		e->user_invalidated = userpage_invalidated;
927 	}
928 
929 	r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
930 				   &duplicates);
931 	if (unlikely(r != 0)) {
932 		if (r != -ERESTARTSYS)
933 			DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
934 		goto out_free_user_pages;
935 	}
936 
937 	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
938 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
939 
940 		e->bo_va = amdgpu_vm_bo_find(vm, bo);
941 	}
942 
943 	amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
944 					  &p->bytes_moved_vis_threshold);
945 	p->bytes_moved = 0;
946 	p->bytes_moved_vis = 0;
947 
948 	r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
949 				      amdgpu_cs_bo_validate, p);
950 	if (r) {
951 		DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
952 		goto error_validate;
953 	}
954 
955 	r = amdgpu_cs_list_validate(p, &duplicates);
956 	if (r)
957 		goto error_validate;
958 
959 	r = amdgpu_cs_list_validate(p, &p->validated);
960 	if (r)
961 		goto error_validate;
962 
963 	if (p->uf_entry.tv.bo) {
964 		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo);
965 
966 		r = amdgpu_ttm_alloc_gart(&uf->tbo);
967 		if (r)
968 			goto error_validate;
969 
970 		p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(uf);
971 	}
972 
973 	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
974 				     p->bytes_moved_vis);
975 
976 	for (i = 0; i < p->gang_size; ++i)
977 		amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,
978 					 p->bo_list->gws_obj,
979 					 p->bo_list->oa_obj);
980 	return 0;
981 
982 error_validate:
983 	ttm_eu_backoff_reservation(&p->ticket, &p->validated);
984 
985 out_free_user_pages:
986 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
987 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
988 
989 		if (!e->user_pages)
990 			continue;
991 		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
992 		kvfree(e->user_pages);
993 		e->user_pages = NULL;
994 	}
995 	return r;
996 }
997 
998 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)
999 {
1000 	int i, j;
1001 
1002 	if (!trace_amdgpu_cs_enabled())
1003 		return;
1004 
1005 	for (i = 0; i < p->gang_size; ++i) {
1006 		struct amdgpu_job *job = p->jobs[i];
1007 
1008 		for (j = 0; j < job->num_ibs; ++j)
1009 			trace_amdgpu_cs(p, job, &job->ibs[j]);
1010 	}
1011 }
1012 
1013 static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
1014 			       struct amdgpu_job *job)
1015 {
1016 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1017 	unsigned int i;
1018 	int r;
1019 
1020 	/* Only for UVD/VCE VM emulation */
1021 	if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)
1022 		return 0;
1023 
1024 	for (i = 0; i < job->num_ibs; ++i) {
1025 		struct amdgpu_ib *ib = &job->ibs[i];
1026 		struct amdgpu_bo_va_mapping *m;
1027 		struct amdgpu_bo *aobj;
1028 		uint64_t va_start;
1029 		uint8_t *kptr;
1030 
1031 		va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;
1032 		r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
1033 		if (r) {
1034 			DRM_ERROR("IB va_start is invalid\n");
1035 			return r;
1036 		}
1037 
1038 		if ((va_start + ib->length_dw * 4) >
1039 		    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
1040 			DRM_ERROR("IB va_start+ib_bytes is invalid\n");
1041 			return -EINVAL;
1042 		}
1043 
1044 		/* the IB should be reserved at this point */
1045 		r = amdgpu_bo_kmap(aobj, (void **)&kptr);
1046 		if (r) {
1047 			return r;
1048 		}
1049 
1050 		kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);
1051 
1052 		if (ring->funcs->parse_cs) {
1053 			memcpy(ib->ptr, kptr, ib->length_dw * 4);
1054 			amdgpu_bo_kunmap(aobj);
1055 
1056 			r = amdgpu_ring_parse_cs(ring, p, job, ib);
1057 			if (r)
1058 				return r;
1059 		} else {
1060 			ib->ptr = (uint32_t *)kptr;
1061 			r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);
1062 			amdgpu_bo_kunmap(aobj);
1063 			if (r)
1064 				return r;
1065 		}
1066 	}
1067 
1068 	return 0;
1069 }
1070 
1071 static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)
1072 {
1073 	unsigned int i;
1074 	int r;
1075 
1076 	for (i = 0; i < p->gang_size; ++i) {
1077 		r = amdgpu_cs_patch_ibs(p, p->jobs[i]);
1078 		if (r)
1079 			return r;
1080 	}
1081 	return 0;
1082 }
1083 
1084 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
1085 {
1086 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1087 	struct amdgpu_job *job = p->gang_leader;
1088 	struct amdgpu_device *adev = p->adev;
1089 	struct amdgpu_vm *vm = &fpriv->vm;
1090 	struct amdgpu_bo_list_entry *e;
1091 	struct amdgpu_bo_va *bo_va;
1092 	struct amdgpu_bo *bo;
1093 	unsigned int i;
1094 	int r;
1095 
1096 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
1097 	if (r)
1098 		return r;
1099 
1100 	r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1101 	if (r)
1102 		return r;
1103 
1104 	r = amdgpu_sync_fence(&job->sync, fpriv->prt_va->last_pt_update);
1105 	if (r)
1106 		return r;
1107 
1108 	if (fpriv->csa_va) {
1109 		bo_va = fpriv->csa_va;
1110 		BUG_ON(!bo_va);
1111 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1112 		if (r)
1113 			return r;
1114 
1115 		r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update);
1116 		if (r)
1117 			return r;
1118 	}
1119 
1120 	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1121 		/* ignore duplicates */
1122 		bo = ttm_to_amdgpu_bo(e->tv.bo);
1123 		if (!bo)
1124 			continue;
1125 
1126 		bo_va = e->bo_va;
1127 		if (bo_va == NULL)
1128 			continue;
1129 
1130 		r = amdgpu_vm_bo_update(adev, bo_va, false);
1131 		if (r)
1132 			return r;
1133 
1134 		r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update);
1135 		if (r)
1136 			return r;
1137 	}
1138 
1139 	r = amdgpu_vm_handle_moved(adev, vm);
1140 	if (r)
1141 		return r;
1142 
1143 	r = amdgpu_vm_update_pdes(adev, vm, false);
1144 	if (r)
1145 		return r;
1146 
1147 	r = amdgpu_sync_fence(&job->sync, vm->last_update);
1148 	if (r)
1149 		return r;
1150 
1151 	for (i = 0; i < p->gang_size; ++i) {
1152 		job = p->jobs[i];
1153 
1154 		if (!job->vm)
1155 			continue;
1156 
1157 		job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
1158 	}
1159 
1160 	if (amdgpu_vm_debug) {
1161 		/* Invalidate all BOs to test for userspace bugs */
1162 		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1163 			struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
1164 
1165 			/* ignore duplicates */
1166 			if (!bo)
1167 				continue;
1168 
1169 			amdgpu_vm_bo_invalidate(adev, bo, false);
1170 		}
1171 	}
1172 
1173 	return 0;
1174 }
1175 
1176 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
1177 {
1178 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1179 	struct amdgpu_job *leader = p->gang_leader;
1180 	struct amdgpu_bo_list_entry *e;
1181 	unsigned int i;
1182 	int r;
1183 
1184 	list_for_each_entry(e, &p->validated, tv.head) {
1185 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
1186 		struct dma_resv *resv = bo->tbo.base.resv;
1187 		enum amdgpu_sync_mode sync_mode;
1188 
1189 		sync_mode = amdgpu_bo_explicit_sync(bo) ?
1190 			AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
1191 		r = amdgpu_sync_resv(p->adev, &leader->sync, resv, sync_mode,
1192 				     &fpriv->vm);
1193 		if (r)
1194 			return r;
1195 	}
1196 
1197 	for (i = 0; i < p->gang_size - 1; ++i) {
1198 		r = amdgpu_sync_clone(&leader->sync, &p->jobs[i]->sync);
1199 		if (r)
1200 			return r;
1201 	}
1202 
1203 	r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_size - 1]);
1204 	if (r && r != -ERESTARTSYS)
1205 		DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");
1206 
1207 	return r;
1208 }
1209 
1210 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
1211 {
1212 	int i;
1213 
1214 	for (i = 0; i < p->num_post_deps; ++i) {
1215 		if (p->post_deps[i].chain && p->post_deps[i].point) {
1216 			drm_syncobj_add_point(p->post_deps[i].syncobj,
1217 					      p->post_deps[i].chain,
1218 					      p->fence, p->post_deps[i].point);
1219 			p->post_deps[i].chain = NULL;
1220 		} else {
1221 			drm_syncobj_replace_fence(p->post_deps[i].syncobj,
1222 						  p->fence);
1223 		}
1224 	}
1225 }
1226 
1227 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1228 			    union drm_amdgpu_cs *cs)
1229 {
1230 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1231 	struct amdgpu_job *leader = p->gang_leader;
1232 	struct amdgpu_bo_list_entry *e;
1233 	unsigned int i;
1234 	uint64_t seq;
1235 	int r;
1236 
1237 	for (i = 0; i < p->gang_size; ++i)
1238 		drm_sched_job_arm(&p->jobs[i]->base);
1239 
1240 	for (i = 0; i < (p->gang_size - 1); ++i) {
1241 		struct dma_fence *fence;
1242 
1243 		fence = &p->jobs[i]->base.s_fence->scheduled;
1244 		r = amdgpu_sync_fence(&leader->sync, fence);
1245 		if (r)
1246 			goto error_cleanup;
1247 	}
1248 
1249 	if (p->gang_size > 1) {
1250 		for (i = 0; i < p->gang_size; ++i)
1251 			amdgpu_job_set_gang_leader(p->jobs[i], leader);
1252 	}
1253 
1254 	/* No memory allocation is allowed while holding the notifier lock.
1255 	 * The lock is held until amdgpu_cs_submit is finished and fence is
1256 	 * added to BOs.
1257 	 */
1258 	mutex_lock(&p->adev->notifier_lock);
1259 
1260 	/* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1261 	 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1262 	 */
1263 	r = 0;
1264 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
1265 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
1266 
1267 		r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
1268 	}
1269 	if (r) {
1270 		r = -EAGAIN;
1271 		goto error_unlock;
1272 	}
1273 
1274 	p->fence = dma_fence_get(&leader->base.s_fence->finished);
1275 	list_for_each_entry(e, &p->validated, tv.head) {
1276 
1277 		/* Everybody except for the gang leader uses READ */
1278 		for (i = 0; i < (p->gang_size - 1); ++i) {
1279 			dma_resv_add_fence(e->tv.bo->base.resv,
1280 					   &p->jobs[i]->base.s_fence->finished,
1281 					   DMA_RESV_USAGE_READ);
1282 		}
1283 
1284 		/* The gang leader is remembered as writer */
1285 		e->tv.num_shared = 0;
1286 	}
1287 
1288 	seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_size - 1],
1289 				   p->fence);
1290 	amdgpu_cs_post_dependencies(p);
1291 
1292 	if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
1293 	    !p->ctx->preamble_presented) {
1294 		leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
1295 		p->ctx->preamble_presented = true;
1296 	}
1297 
1298 	cs->out.handle = seq;
1299 	leader->uf_sequence = seq;
1300 
1301 	amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket);
1302 	for (i = 0; i < p->gang_size; ++i) {
1303 		amdgpu_job_free_resources(p->jobs[i]);
1304 		trace_amdgpu_cs_ioctl(p->jobs[i]);
1305 		drm_sched_entity_push_job(&p->jobs[i]->base);
1306 		p->jobs[i] = NULL;
1307 	}
1308 
1309 	amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
1310 	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
1311 
1312 	mutex_unlock(&p->adev->notifier_lock);
1313 	mutex_unlock(&p->bo_list->bo_list_mutex);
1314 	return 0;
1315 
1316 error_unlock:
1317 	mutex_unlock(&p->adev->notifier_lock);
1318 
1319 error_cleanup:
1320 	for (i = 0; i < p->gang_size; ++i)
1321 		drm_sched_job_cleanup(&p->jobs[i]->base);
1322 	return r;
1323 }
1324 
1325 /* Cleanup the parser structure */
1326 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)
1327 {
1328 	unsigned i;
1329 
1330 	for (i = 0; i < parser->num_post_deps; i++) {
1331 		drm_syncobj_put(parser->post_deps[i].syncobj);
1332 		kfree(parser->post_deps[i].chain);
1333 	}
1334 	kfree(parser->post_deps);
1335 
1336 	dma_fence_put(parser->fence);
1337 
1338 	if (parser->ctx)
1339 		amdgpu_ctx_put(parser->ctx);
1340 	if (parser->bo_list)
1341 		amdgpu_bo_list_put(parser->bo_list);
1342 
1343 	for (i = 0; i < parser->nchunks; i++)
1344 		kvfree(parser->chunks[i].kdata);
1345 	kvfree(parser->chunks);
1346 	for (i = 0; i < parser->gang_size; ++i) {
1347 		if (parser->jobs[i])
1348 			amdgpu_job_free(parser->jobs[i]);
1349 	}
1350 	if (parser->uf_entry.tv.bo) {
1351 		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo);
1352 
1353 		amdgpu_bo_unref(&uf);
1354 	}
1355 }
1356 
1357 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1358 {
1359 	struct amdgpu_device *adev = drm_to_adev(dev);
1360 	struct amdgpu_cs_parser parser;
1361 	int r;
1362 
1363 	if (amdgpu_ras_intr_triggered())
1364 		return -EHWPOISON;
1365 
1366 	if (!adev->accel_working)
1367 		return -EBUSY;
1368 
1369 	r = amdgpu_cs_parser_init(&parser, adev, filp, data);
1370 	if (r) {
1371 		if (printk_ratelimit())
1372 			DRM_ERROR("Failed to initialize parser %d!\n", r);
1373 		return r;
1374 	}
1375 
1376 	r = amdgpu_cs_pass1(&parser, data);
1377 	if (r)
1378 		goto error_fini;
1379 
1380 	r = amdgpu_cs_pass2(&parser);
1381 	if (r)
1382 		goto error_fini;
1383 
1384 	r = amdgpu_cs_parser_bos(&parser, data);
1385 	if (r) {
1386 		if (r == -ENOMEM)
1387 			DRM_ERROR("Not enough memory for command submission!\n");
1388 		else if (r != -ERESTARTSYS && r != -EAGAIN)
1389 			DRM_ERROR("Failed to process the buffer list %d!\n", r);
1390 		goto error_fini;
1391 	}
1392 
1393 	r = amdgpu_cs_patch_jobs(&parser);
1394 	if (r)
1395 		goto error_backoff;
1396 
1397 	r = amdgpu_cs_vm_handling(&parser);
1398 	if (r)
1399 		goto error_backoff;
1400 
1401 	r = amdgpu_cs_sync_rings(&parser);
1402 	if (r)
1403 		goto error_backoff;
1404 
1405 	trace_amdgpu_cs_ibs(&parser);
1406 
1407 	r = amdgpu_cs_submit(&parser, data);
1408 	if (r)
1409 		goto error_backoff;
1410 
1411 	amdgpu_cs_parser_fini(&parser);
1412 	return 0;
1413 
1414 error_backoff:
1415 	ttm_eu_backoff_reservation(&parser.ticket, &parser.validated);
1416 	mutex_unlock(&parser.bo_list->bo_list_mutex);
1417 
1418 error_fini:
1419 	amdgpu_cs_parser_fini(&parser);
1420 	return r;
1421 }
1422 
1423 /**
1424  * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1425  *
1426  * @dev: drm device
1427  * @data: data from userspace
1428  * @filp: file private
1429  *
1430  * Wait for the command submission identified by handle to finish.
1431  */
1432 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
1433 			 struct drm_file *filp)
1434 {
1435 	union drm_amdgpu_wait_cs *wait = data;
1436 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1437 	struct drm_sched_entity *entity;
1438 	struct amdgpu_ctx *ctx;
1439 	struct dma_fence *fence;
1440 	long r;
1441 
1442 	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
1443 	if (ctx == NULL)
1444 		return -EINVAL;
1445 
1446 	r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,
1447 				  wait->in.ring, &entity);
1448 	if (r) {
1449 		amdgpu_ctx_put(ctx);
1450 		return r;
1451 	}
1452 
1453 	fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);
1454 	if (IS_ERR(fence))
1455 		r = PTR_ERR(fence);
1456 	else if (fence) {
1457 		r = dma_fence_wait_timeout(fence, true, timeout);
1458 		if (r > 0 && fence->error)
1459 			r = fence->error;
1460 		dma_fence_put(fence);
1461 	} else
1462 		r = 1;
1463 
1464 	amdgpu_ctx_put(ctx);
1465 	if (r < 0)
1466 		return r;
1467 
1468 	memset(wait, 0, sizeof(*wait));
1469 	wait->out.status = (r == 0);
1470 
1471 	return 0;
1472 }
1473 
1474 /**
1475  * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1476  *
1477  * @adev: amdgpu device
1478  * @filp: file private
1479  * @user: drm_amdgpu_fence copied from user space
1480  */
1481 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1482 					     struct drm_file *filp,
1483 					     struct drm_amdgpu_fence *user)
1484 {
1485 	struct drm_sched_entity *entity;
1486 	struct amdgpu_ctx *ctx;
1487 	struct dma_fence *fence;
1488 	int r;
1489 
1490 	ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
1491 	if (ctx == NULL)
1492 		return ERR_PTR(-EINVAL);
1493 
1494 	r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,
1495 				  user->ring, &entity);
1496 	if (r) {
1497 		amdgpu_ctx_put(ctx);
1498 		return ERR_PTR(r);
1499 	}
1500 
1501 	fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);
1502 	amdgpu_ctx_put(ctx);
1503 
1504 	return fence;
1505 }
1506 
1507 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1508 				    struct drm_file *filp)
1509 {
1510 	struct amdgpu_device *adev = drm_to_adev(dev);
1511 	union drm_amdgpu_fence_to_handle *info = data;
1512 	struct dma_fence *fence;
1513 	struct drm_syncobj *syncobj;
1514 	struct sync_file *sync_file;
1515 	int fd, r;
1516 
1517 	fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1518 	if (IS_ERR(fence))
1519 		return PTR_ERR(fence);
1520 
1521 	if (!fence)
1522 		fence = dma_fence_get_stub();
1523 
1524 	switch (info->in.what) {
1525 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1526 		r = drm_syncobj_create(&syncobj, 0, fence);
1527 		dma_fence_put(fence);
1528 		if (r)
1529 			return r;
1530 		r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1531 		drm_syncobj_put(syncobj);
1532 		return r;
1533 
1534 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1535 		r = drm_syncobj_create(&syncobj, 0, fence);
1536 		dma_fence_put(fence);
1537 		if (r)
1538 			return r;
1539 		r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);
1540 		drm_syncobj_put(syncobj);
1541 		return r;
1542 
1543 	case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1544 		fd = get_unused_fd_flags(O_CLOEXEC);
1545 		if (fd < 0) {
1546 			dma_fence_put(fence);
1547 			return fd;
1548 		}
1549 
1550 		sync_file = sync_file_create(fence);
1551 		dma_fence_put(fence);
1552 		if (!sync_file) {
1553 			put_unused_fd(fd);
1554 			return -ENOMEM;
1555 		}
1556 
1557 		fd_install(fd, sync_file->file);
1558 		info->out.handle = fd;
1559 		return 0;
1560 
1561 	default:
1562 		dma_fence_put(fence);
1563 		return -EINVAL;
1564 	}
1565 }
1566 
1567 /**
1568  * amdgpu_cs_wait_all_fences - wait on all fences to signal
1569  *
1570  * @adev: amdgpu device
1571  * @filp: file private
1572  * @wait: wait parameters
1573  * @fences: array of drm_amdgpu_fence
1574  */
1575 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
1576 				     struct drm_file *filp,
1577 				     union drm_amdgpu_wait_fences *wait,
1578 				     struct drm_amdgpu_fence *fences)
1579 {
1580 	uint32_t fence_count = wait->in.fence_count;
1581 	unsigned int i;
1582 	long r = 1;
1583 
1584 	for (i = 0; i < fence_count; i++) {
1585 		struct dma_fence *fence;
1586 		unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1587 
1588 		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1589 		if (IS_ERR(fence))
1590 			return PTR_ERR(fence);
1591 		else if (!fence)
1592 			continue;
1593 
1594 		r = dma_fence_wait_timeout(fence, true, timeout);
1595 		dma_fence_put(fence);
1596 		if (r < 0)
1597 			return r;
1598 
1599 		if (r == 0)
1600 			break;
1601 
1602 		if (fence->error)
1603 			return fence->error;
1604 	}
1605 
1606 	memset(wait, 0, sizeof(*wait));
1607 	wait->out.status = (r > 0);
1608 
1609 	return 0;
1610 }
1611 
1612 /**
1613  * amdgpu_cs_wait_any_fence - wait on any fence to signal
1614  *
1615  * @adev: amdgpu device
1616  * @filp: file private
1617  * @wait: wait parameters
1618  * @fences: array of drm_amdgpu_fence
1619  */
1620 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
1621 				    struct drm_file *filp,
1622 				    union drm_amdgpu_wait_fences *wait,
1623 				    struct drm_amdgpu_fence *fences)
1624 {
1625 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1626 	uint32_t fence_count = wait->in.fence_count;
1627 	uint32_t first = ~0;
1628 	struct dma_fence **array;
1629 	unsigned int i;
1630 	long r;
1631 
1632 	/* Prepare the fence array */
1633 	array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
1634 
1635 	if (array == NULL)
1636 		return -ENOMEM;
1637 
1638 	for (i = 0; i < fence_count; i++) {
1639 		struct dma_fence *fence;
1640 
1641 		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1642 		if (IS_ERR(fence)) {
1643 			r = PTR_ERR(fence);
1644 			goto err_free_fence_array;
1645 		} else if (fence) {
1646 			array[i] = fence;
1647 		} else { /* NULL, the fence has been already signaled */
1648 			r = 1;
1649 			first = i;
1650 			goto out;
1651 		}
1652 	}
1653 
1654 	r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
1655 				       &first);
1656 	if (r < 0)
1657 		goto err_free_fence_array;
1658 
1659 out:
1660 	memset(wait, 0, sizeof(*wait));
1661 	wait->out.status = (r > 0);
1662 	wait->out.first_signaled = first;
1663 
1664 	if (first < fence_count && array[first])
1665 		r = array[first]->error;
1666 	else
1667 		r = 0;
1668 
1669 err_free_fence_array:
1670 	for (i = 0; i < fence_count; i++)
1671 		dma_fence_put(array[i]);
1672 	kfree(array);
1673 
1674 	return r;
1675 }
1676 
1677 /**
1678  * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1679  *
1680  * @dev: drm device
1681  * @data: data from userspace
1682  * @filp: file private
1683  */
1684 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1685 				struct drm_file *filp)
1686 {
1687 	struct amdgpu_device *adev = drm_to_adev(dev);
1688 	union drm_amdgpu_wait_fences *wait = data;
1689 	uint32_t fence_count = wait->in.fence_count;
1690 	struct drm_amdgpu_fence *fences_user;
1691 	struct drm_amdgpu_fence *fences;
1692 	int r;
1693 
1694 	/* Get the fences from userspace */
1695 	fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
1696 			GFP_KERNEL);
1697 	if (fences == NULL)
1698 		return -ENOMEM;
1699 
1700 	fences_user = u64_to_user_ptr(wait->in.fences);
1701 	if (copy_from_user(fences, fences_user,
1702 		sizeof(struct drm_amdgpu_fence) * fence_count)) {
1703 		r = -EFAULT;
1704 		goto err_free_fences;
1705 	}
1706 
1707 	if (wait->in.wait_all)
1708 		r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
1709 	else
1710 		r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
1711 
1712 err_free_fences:
1713 	kfree(fences);
1714 
1715 	return r;
1716 }
1717 
1718 /**
1719  * amdgpu_cs_find_mapping - find bo_va for VM address
1720  *
1721  * @parser: command submission parser context
1722  * @addr: VM address
1723  * @bo: resulting BO of the mapping found
1724  * @map: Placeholder to return found BO mapping
1725  *
1726  * Search the buffer objects in the command submission context for a certain
1727  * virtual memory address. Returns allocation structure when found, NULL
1728  * otherwise.
1729  */
1730 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1731 			   uint64_t addr, struct amdgpu_bo **bo,
1732 			   struct amdgpu_bo_va_mapping **map)
1733 {
1734 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1735 	struct ttm_operation_ctx ctx = { false, false };
1736 	struct amdgpu_vm *vm = &fpriv->vm;
1737 	struct amdgpu_bo_va_mapping *mapping;
1738 	int r;
1739 
1740 	addr /= AMDGPU_GPU_PAGE_SIZE;
1741 
1742 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
1743 	if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1744 		return -EINVAL;
1745 
1746 	*bo = mapping->bo_va->base.bo;
1747 	*map = mapping;
1748 
1749 	/* Double check that the BO is reserved by this CS */
1750 	if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->ticket)
1751 		return -EINVAL;
1752 
1753 	if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
1754 		(*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1755 		amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1756 		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1757 		if (r)
1758 			return r;
1759 	}
1760 
1761 	return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1762 }
1763