1 /*
2  * Copyright 2008 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22  * DEALINGS IN THE SOFTWARE.
23  *
24  * Authors:
25  *    Jerome Glisse <glisse@freedesktop.org>
26  */
27 #include <linux/pagemap.h>
28 #include <drm/drmP.h>
29 #include <drm/amdgpu_drm.h>
30 #include "amdgpu.h"
31 #include "amdgpu_trace.h"
32 
33 int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
34 		       u32 ip_instance, u32 ring,
35 		       struct amdgpu_ring **out_ring)
36 {
37 	/* Right now all IPs have only one instance - multiple rings. */
38 	if (ip_instance != 0) {
39 		DRM_ERROR("invalid ip instance: %d\n", ip_instance);
40 		return -EINVAL;
41 	}
42 
43 	switch (ip_type) {
44 	default:
45 		DRM_ERROR("unknown ip type: %d\n", ip_type);
46 		return -EINVAL;
47 	case AMDGPU_HW_IP_GFX:
48 		if (ring < adev->gfx.num_gfx_rings) {
49 			*out_ring = &adev->gfx.gfx_ring[ring];
50 		} else {
51 			DRM_ERROR("only %d gfx rings are supported now\n",
52 				  adev->gfx.num_gfx_rings);
53 			return -EINVAL;
54 		}
55 		break;
56 	case AMDGPU_HW_IP_COMPUTE:
57 		if (ring < adev->gfx.num_compute_rings) {
58 			*out_ring = &adev->gfx.compute_ring[ring];
59 		} else {
60 			DRM_ERROR("only %d compute rings are supported now\n",
61 				  adev->gfx.num_compute_rings);
62 			return -EINVAL;
63 		}
64 		break;
65 	case AMDGPU_HW_IP_DMA:
66 		if (ring < adev->sdma.num_instances) {
67 			*out_ring = &adev->sdma.instance[ring].ring;
68 		} else {
69 			DRM_ERROR("only %d SDMA rings are supported\n",
70 				  adev->sdma.num_instances);
71 			return -EINVAL;
72 		}
73 		break;
74 	case AMDGPU_HW_IP_UVD:
75 		*out_ring = &adev->uvd.ring;
76 		break;
77 	case AMDGPU_HW_IP_VCE:
78 		if (ring < 2){
79 			*out_ring = &adev->vce.ring[ring];
80 		} else {
81 			DRM_ERROR("only two VCE rings are supported\n");
82 			return -EINVAL;
83 		}
84 		break;
85 	}
86 	return 0;
87 }
88 
89 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
90 				      struct drm_amdgpu_cs_chunk_fence *data,
91 				      uint32_t *offset)
92 {
93 	struct drm_gem_object *gobj;
94 
95 	gobj = drm_gem_object_lookup(p->filp, data->handle);
96 	if (gobj == NULL)
97 		return -EINVAL;
98 
99 	p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
100 	p->uf_entry.priority = 0;
101 	p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
102 	p->uf_entry.tv.shared = true;
103 	p->uf_entry.user_pages = NULL;
104 	*offset = data->offset;
105 
106 	drm_gem_object_unreference_unlocked(gobj);
107 
108 	if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
109 		amdgpu_bo_unref(&p->uf_entry.robj);
110 		return -EINVAL;
111 	}
112 
113 	return 0;
114 }
115 
116 int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
117 {
118 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
119 	struct amdgpu_vm *vm = &fpriv->vm;
120 	union drm_amdgpu_cs *cs = data;
121 	uint64_t *chunk_array_user;
122 	uint64_t *chunk_array;
123 	unsigned size, num_ibs = 0;
124 	uint32_t uf_offset = 0;
125 	int i;
126 	int ret;
127 
128 	if (cs->in.num_chunks == 0)
129 		return 0;
130 
131 	chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
132 	if (!chunk_array)
133 		return -ENOMEM;
134 
135 	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
136 	if (!p->ctx) {
137 		ret = -EINVAL;
138 		goto free_chunk;
139 	}
140 
141 	/* get chunks */
142 	chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
143 	if (copy_from_user(chunk_array, chunk_array_user,
144 			   sizeof(uint64_t)*cs->in.num_chunks)) {
145 		ret = -EFAULT;
146 		goto put_ctx;
147 	}
148 
149 	p->nchunks = cs->in.num_chunks;
150 	p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
151 			    GFP_KERNEL);
152 	if (!p->chunks) {
153 		ret = -ENOMEM;
154 		goto put_ctx;
155 	}
156 
157 	for (i = 0; i < p->nchunks; i++) {
158 		struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
159 		struct drm_amdgpu_cs_chunk user_chunk;
160 		uint32_t __user *cdata;
161 
162 		chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
163 		if (copy_from_user(&user_chunk, chunk_ptr,
164 				       sizeof(struct drm_amdgpu_cs_chunk))) {
165 			ret = -EFAULT;
166 			i--;
167 			goto free_partial_kdata;
168 		}
169 		p->chunks[i].chunk_id = user_chunk.chunk_id;
170 		p->chunks[i].length_dw = user_chunk.length_dw;
171 
172 		size = p->chunks[i].length_dw;
173 		cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
174 
175 		p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
176 		if (p->chunks[i].kdata == NULL) {
177 			ret = -ENOMEM;
178 			i--;
179 			goto free_partial_kdata;
180 		}
181 		size *= sizeof(uint32_t);
182 		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
183 			ret = -EFAULT;
184 			goto free_partial_kdata;
185 		}
186 
187 		switch (p->chunks[i].chunk_id) {
188 		case AMDGPU_CHUNK_ID_IB:
189 			++num_ibs;
190 			break;
191 
192 		case AMDGPU_CHUNK_ID_FENCE:
193 			size = sizeof(struct drm_amdgpu_cs_chunk_fence);
194 			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
195 				ret = -EINVAL;
196 				goto free_partial_kdata;
197 			}
198 
199 			ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
200 							 &uf_offset);
201 			if (ret)
202 				goto free_partial_kdata;
203 
204 			break;
205 
206 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
207 			break;
208 
209 		default:
210 			ret = -EINVAL;
211 			goto free_partial_kdata;
212 		}
213 	}
214 
215 	ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
216 	if (ret)
217 		goto free_all_kdata;
218 
219 	if (p->uf_entry.robj) {
220 		p->job->uf_bo = amdgpu_bo_ref(p->uf_entry.robj);
221 		p->job->uf_offset = uf_offset;
222 	}
223 
224 	kfree(chunk_array);
225 	return 0;
226 
227 free_all_kdata:
228 	i = p->nchunks - 1;
229 free_partial_kdata:
230 	for (; i >= 0; i--)
231 		drm_free_large(p->chunks[i].kdata);
232 	kfree(p->chunks);
233 put_ctx:
234 	amdgpu_ctx_put(p->ctx);
235 free_chunk:
236 	kfree(chunk_array);
237 
238 	return ret;
239 }
240 
241 /* Returns how many bytes TTM can move per IB.
242  */
243 static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
244 {
245 	u64 real_vram_size = adev->mc.real_vram_size;
246 	u64 vram_usage = atomic64_read(&adev->vram_usage);
247 
248 	/* This function is based on the current VRAM usage.
249 	 *
250 	 * - If all of VRAM is free, allow relocating the number of bytes that
251 	 *   is equal to 1/4 of the size of VRAM for this IB.
252 
253 	 * - If more than one half of VRAM is occupied, only allow relocating
254 	 *   1 MB of data for this IB.
255 	 *
256 	 * - From 0 to one half of used VRAM, the threshold decreases
257 	 *   linearly.
258 	 *         __________________
259 	 * 1/4 of -|\               |
260 	 * VRAM    | \              |
261 	 *         |  \             |
262 	 *         |   \            |
263 	 *         |    \           |
264 	 *         |     \          |
265 	 *         |      \         |
266 	 *         |       \________|1 MB
267 	 *         |----------------|
268 	 *    VRAM 0 %             100 %
269 	 *         used            used
270 	 *
271 	 * Note: It's a threshold, not a limit. The threshold must be crossed
272 	 * for buffer relocations to stop, so any buffer of an arbitrary size
273 	 * can be moved as long as the threshold isn't crossed before
274 	 * the relocation takes place. We don't want to disable buffer
275 	 * relocations completely.
276 	 *
277 	 * The idea is that buffers should be placed in VRAM at creation time
278 	 * and TTM should only do a minimum number of relocations during
279 	 * command submission. In practice, you need to submit at least
280 	 * a dozen IBs to move all buffers to VRAM if they are in GTT.
281 	 *
282 	 * Also, things can get pretty crazy under memory pressure and actual
283 	 * VRAM usage can change a lot, so playing safe even at 50% does
284 	 * consistently increase performance.
285 	 */
286 
287 	u64 half_vram = real_vram_size >> 1;
288 	u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
289 	u64 bytes_moved_threshold = half_free_vram >> 1;
290 	return max(bytes_moved_threshold, 1024*1024ull);
291 }
292 
293 int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
294 			    struct list_head *validated)
295 {
296 	struct amdgpu_bo_list_entry *lobj;
297 	u64 initial_bytes_moved;
298 	int r;
299 
300 	list_for_each_entry(lobj, validated, tv.head) {
301 		struct amdgpu_bo *bo = lobj->robj;
302 		bool binding_userptr = false;
303 		struct mm_struct *usermm;
304 		uint32_t domain;
305 
306 		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
307 		if (usermm && usermm != current->mm)
308 			return -EPERM;
309 
310 		/* Check if we have user pages and nobody bound the BO already */
311 		if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
312 			size_t size = sizeof(struct page *);
313 
314 			size *= bo->tbo.ttm->num_pages;
315 			memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
316 			binding_userptr = true;
317 		}
318 
319 		if (bo->pin_count)
320 			continue;
321 
322 		/* Avoid moving this one if we have moved too many buffers
323 		 * for this IB already.
324 		 *
325 		 * Note that this allows moving at least one buffer of
326 		 * any size, because it doesn't take the current "bo"
327 		 * into account. We don't want to disallow buffer moves
328 		 * completely.
329 		 */
330 		if (p->bytes_moved <= p->bytes_moved_threshold)
331 			domain = bo->prefered_domains;
332 		else
333 			domain = bo->allowed_domains;
334 
335 	retry:
336 		amdgpu_ttm_placement_from_domain(bo, domain);
337 		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
338 		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
339 		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
340 			       initial_bytes_moved;
341 
342 		if (unlikely(r)) {
343 			if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
344 				domain = bo->allowed_domains;
345 				goto retry;
346 			}
347 			return r;
348 		}
349 
350 		if (binding_userptr) {
351 			drm_free_large(lobj->user_pages);
352 			lobj->user_pages = NULL;
353 		}
354 	}
355 	return 0;
356 }
357 
358 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
359 				union drm_amdgpu_cs *cs)
360 {
361 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
362 	struct amdgpu_bo_list_entry *e;
363 	struct list_head duplicates;
364 	bool need_mmap_lock = false;
365 	unsigned i, tries = 10;
366 	int r;
367 
368 	INIT_LIST_HEAD(&p->validated);
369 
370 	p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
371 	if (p->bo_list) {
372 		need_mmap_lock = p->bo_list->first_userptr !=
373 			p->bo_list->num_entries;
374 		amdgpu_bo_list_get_list(p->bo_list, &p->validated);
375 	}
376 
377 	INIT_LIST_HEAD(&duplicates);
378 	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
379 
380 	if (p->uf_entry.robj)
381 		list_add(&p->uf_entry.tv.head, &p->validated);
382 
383 	if (need_mmap_lock)
384 		down_read(&current->mm->mmap_sem);
385 
386 	while (1) {
387 		struct list_head need_pages;
388 		unsigned i;
389 
390 		r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
391 					   &duplicates);
392 		if (unlikely(r != 0))
393 			goto error_free_pages;
394 
395 		/* Without a BO list we don't have userptr BOs */
396 		if (!p->bo_list)
397 			break;
398 
399 		INIT_LIST_HEAD(&need_pages);
400 		for (i = p->bo_list->first_userptr;
401 		     i < p->bo_list->num_entries; ++i) {
402 
403 			e = &p->bo_list->array[i];
404 
405 			if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
406 				 &e->user_invalidated) && e->user_pages) {
407 
408 				/* We acquired a page array, but somebody
409 				 * invalidated it. Free it an try again
410 				 */
411 				release_pages(e->user_pages,
412 					      e->robj->tbo.ttm->num_pages,
413 					      false);
414 				drm_free_large(e->user_pages);
415 				e->user_pages = NULL;
416 			}
417 
418 			if (e->robj->tbo.ttm->state != tt_bound &&
419 			    !e->user_pages) {
420 				list_del(&e->tv.head);
421 				list_add(&e->tv.head, &need_pages);
422 
423 				amdgpu_bo_unreserve(e->robj);
424 			}
425 		}
426 
427 		if (list_empty(&need_pages))
428 			break;
429 
430 		/* Unreserve everything again. */
431 		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
432 
433 		/* We tried to often, just abort */
434 		if (!--tries) {
435 			r = -EDEADLK;
436 			goto error_free_pages;
437 		}
438 
439 		/* Fill the page arrays for all useptrs. */
440 		list_for_each_entry(e, &need_pages, tv.head) {
441 			struct ttm_tt *ttm = e->robj->tbo.ttm;
442 
443 			e->user_pages = drm_calloc_large(ttm->num_pages,
444 							 sizeof(struct page*));
445 			if (!e->user_pages) {
446 				r = -ENOMEM;
447 				goto error_free_pages;
448 			}
449 
450 			r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
451 			if (r) {
452 				drm_free_large(e->user_pages);
453 				e->user_pages = NULL;
454 				goto error_free_pages;
455 			}
456 		}
457 
458 		/* And try again. */
459 		list_splice(&need_pages, &p->validated);
460 	}
461 
462 	amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
463 
464 	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
465 	p->bytes_moved = 0;
466 
467 	r = amdgpu_cs_list_validate(p, &duplicates);
468 	if (r)
469 		goto error_validate;
470 
471 	r = amdgpu_cs_list_validate(p, &p->validated);
472 	if (r)
473 		goto error_validate;
474 
475 	if (p->bo_list) {
476 		struct amdgpu_bo *gds = p->bo_list->gds_obj;
477 		struct amdgpu_bo *gws = p->bo_list->gws_obj;
478 		struct amdgpu_bo *oa = p->bo_list->oa_obj;
479 		struct amdgpu_vm *vm = &fpriv->vm;
480 		unsigned i;
481 
482 		for (i = 0; i < p->bo_list->num_entries; i++) {
483 			struct amdgpu_bo *bo = p->bo_list->array[i].robj;
484 
485 			p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
486 		}
487 
488 		if (gds) {
489 			p->job->gds_base = amdgpu_bo_gpu_offset(gds);
490 			p->job->gds_size = amdgpu_bo_size(gds);
491 		}
492 		if (gws) {
493 			p->job->gws_base = amdgpu_bo_gpu_offset(gws);
494 			p->job->gws_size = amdgpu_bo_size(gws);
495 		}
496 		if (oa) {
497 			p->job->oa_base = amdgpu_bo_gpu_offset(oa);
498 			p->job->oa_size = amdgpu_bo_size(oa);
499 		}
500 	}
501 
502 error_validate:
503 	if (r) {
504 		amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
505 		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
506 	}
507 
508 error_free_pages:
509 
510 	if (need_mmap_lock)
511 		up_read(&current->mm->mmap_sem);
512 
513 	if (p->bo_list) {
514 		for (i = p->bo_list->first_userptr;
515 		     i < p->bo_list->num_entries; ++i) {
516 			e = &p->bo_list->array[i];
517 
518 			if (!e->user_pages)
519 				continue;
520 
521 			release_pages(e->user_pages,
522 				      e->robj->tbo.ttm->num_pages,
523 				      false);
524 			drm_free_large(e->user_pages);
525 		}
526 	}
527 
528 	return r;
529 }
530 
531 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
532 {
533 	struct amdgpu_bo_list_entry *e;
534 	int r;
535 
536 	list_for_each_entry(e, &p->validated, tv.head) {
537 		struct reservation_object *resv = e->robj->tbo.resv;
538 		r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
539 
540 		if (r)
541 			return r;
542 	}
543 	return 0;
544 }
545 
546 /**
547  * cs_parser_fini() - clean parser states
548  * @parser:	parser structure holding parsing context.
549  * @error:	error number
550  *
551  * If error is set than unvalidate buffer, otherwise just free memory
552  * used by parsing context.
553  **/
554 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
555 {
556 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
557 	unsigned i;
558 
559 	if (!error) {
560 		amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
561 
562 		ttm_eu_fence_buffer_objects(&parser->ticket,
563 					    &parser->validated,
564 					    parser->fence);
565 	} else if (backoff) {
566 		ttm_eu_backoff_reservation(&parser->ticket,
567 					   &parser->validated);
568 	}
569 	fence_put(parser->fence);
570 
571 	if (parser->ctx)
572 		amdgpu_ctx_put(parser->ctx);
573 	if (parser->bo_list)
574 		amdgpu_bo_list_put(parser->bo_list);
575 
576 	for (i = 0; i < parser->nchunks; i++)
577 		drm_free_large(parser->chunks[i].kdata);
578 	kfree(parser->chunks);
579 	if (parser->job)
580 		amdgpu_job_free(parser->job);
581 	amdgpu_bo_unref(&parser->uf_entry.robj);
582 }
583 
584 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
585 				   struct amdgpu_vm *vm)
586 {
587 	struct amdgpu_device *adev = p->adev;
588 	struct amdgpu_bo_va *bo_va;
589 	struct amdgpu_bo *bo;
590 	int i, r;
591 
592 	r = amdgpu_vm_update_page_directory(adev, vm);
593 	if (r)
594 		return r;
595 
596 	r = amdgpu_sync_fence(adev, &p->job->sync, vm->page_directory_fence);
597 	if (r)
598 		return r;
599 
600 	r = amdgpu_vm_clear_freed(adev, vm);
601 	if (r)
602 		return r;
603 
604 	if (p->bo_list) {
605 		for (i = 0; i < p->bo_list->num_entries; i++) {
606 			struct fence *f;
607 
608 			/* ignore duplicates */
609 			bo = p->bo_list->array[i].robj;
610 			if (!bo)
611 				continue;
612 
613 			bo_va = p->bo_list->array[i].bo_va;
614 			if (bo_va == NULL)
615 				continue;
616 
617 			r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem);
618 			if (r)
619 				return r;
620 
621 			f = bo_va->last_pt_update;
622 			r = amdgpu_sync_fence(adev, &p->job->sync, f);
623 			if (r)
624 				return r;
625 		}
626 
627 	}
628 
629 	r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
630 
631 	if (amdgpu_vm_debug && p->bo_list) {
632 		/* Invalidate all BOs to test for userspace bugs */
633 		for (i = 0; i < p->bo_list->num_entries; i++) {
634 			/* ignore duplicates */
635 			bo = p->bo_list->array[i].robj;
636 			if (!bo)
637 				continue;
638 
639 			amdgpu_vm_bo_invalidate(adev, bo);
640 		}
641 	}
642 
643 	return r;
644 }
645 
646 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
647 				 struct amdgpu_cs_parser *p)
648 {
649 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
650 	struct amdgpu_vm *vm = &fpriv->vm;
651 	struct amdgpu_ring *ring = p->job->ring;
652 	int i, r;
653 
654 	/* Only for UVD/VCE VM emulation */
655 	if (ring->funcs->parse_cs) {
656 		for (i = 0; i < p->job->num_ibs; i++) {
657 			r = amdgpu_ring_parse_cs(ring, p, i);
658 			if (r)
659 				return r;
660 		}
661 	}
662 
663 	r = amdgpu_bo_vm_update_pte(p, vm);
664 	if (!r)
665 		amdgpu_cs_sync_rings(p);
666 
667 	return r;
668 }
669 
670 static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
671 {
672 	if (r == -EDEADLK) {
673 		r = amdgpu_gpu_reset(adev);
674 		if (!r)
675 			r = -EAGAIN;
676 	}
677 	return r;
678 }
679 
680 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
681 			     struct amdgpu_cs_parser *parser)
682 {
683 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
684 	struct amdgpu_vm *vm = &fpriv->vm;
685 	int i, j;
686 	int r;
687 
688 	for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
689 		struct amdgpu_cs_chunk *chunk;
690 		struct amdgpu_ib *ib;
691 		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
692 		struct amdgpu_ring *ring;
693 
694 		chunk = &parser->chunks[i];
695 		ib = &parser->job->ibs[j];
696 		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
697 
698 		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
699 			continue;
700 
701 		r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
702 				       chunk_ib->ip_instance, chunk_ib->ring,
703 				       &ring);
704 		if (r)
705 			return r;
706 
707 		if (parser->job->ring && parser->job->ring != ring)
708 			return -EINVAL;
709 
710 		parser->job->ring = ring;
711 
712 		if (ring->funcs->parse_cs) {
713 			struct amdgpu_bo_va_mapping *m;
714 			struct amdgpu_bo *aobj = NULL;
715 			uint64_t offset;
716 			uint8_t *kptr;
717 
718 			m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
719 						   &aobj);
720 			if (!aobj) {
721 				DRM_ERROR("IB va_start is invalid\n");
722 				return -EINVAL;
723 			}
724 
725 			if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
726 			    (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
727 				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
728 				return -EINVAL;
729 			}
730 
731 			/* the IB should be reserved at this point */
732 			r = amdgpu_bo_kmap(aobj, (void **)&kptr);
733 			if (r) {
734 				return r;
735 			}
736 
737 			offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
738 			kptr += chunk_ib->va_start - offset;
739 
740 			r =  amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
741 			if (r) {
742 				DRM_ERROR("Failed to get ib !\n");
743 				return r;
744 			}
745 
746 			memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
747 			amdgpu_bo_kunmap(aobj);
748 		} else {
749 			r =  amdgpu_ib_get(adev, vm, 0, ib);
750 			if (r) {
751 				DRM_ERROR("Failed to get ib !\n");
752 				return r;
753 			}
754 
755 			ib->gpu_addr = chunk_ib->va_start;
756 		}
757 
758 		ib->length_dw = chunk_ib->ib_bytes / 4;
759 		ib->flags = chunk_ib->flags;
760 		j++;
761 	}
762 
763 	/* UVD & VCE fw doesn't support user fences */
764 	if (parser->job->uf_bo && (
765 	    parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
766 	    parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
767 		return -EINVAL;
768 
769 	return 0;
770 }
771 
772 static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
773 				  struct amdgpu_cs_parser *p)
774 {
775 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
776 	int i, j, r;
777 
778 	for (i = 0; i < p->nchunks; ++i) {
779 		struct drm_amdgpu_cs_chunk_dep *deps;
780 		struct amdgpu_cs_chunk *chunk;
781 		unsigned num_deps;
782 
783 		chunk = &p->chunks[i];
784 
785 		if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
786 			continue;
787 
788 		deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
789 		num_deps = chunk->length_dw * 4 /
790 			sizeof(struct drm_amdgpu_cs_chunk_dep);
791 
792 		for (j = 0; j < num_deps; ++j) {
793 			struct amdgpu_ring *ring;
794 			struct amdgpu_ctx *ctx;
795 			struct fence *fence;
796 
797 			r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
798 					       deps[j].ip_instance,
799 					       deps[j].ring, &ring);
800 			if (r)
801 				return r;
802 
803 			ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
804 			if (ctx == NULL)
805 				return -EINVAL;
806 
807 			fence = amdgpu_ctx_get_fence(ctx, ring,
808 						     deps[j].handle);
809 			if (IS_ERR(fence)) {
810 				r = PTR_ERR(fence);
811 				amdgpu_ctx_put(ctx);
812 				return r;
813 
814 			} else if (fence) {
815 				r = amdgpu_sync_fence(adev, &p->job->sync,
816 						      fence);
817 				fence_put(fence);
818 				amdgpu_ctx_put(ctx);
819 				if (r)
820 					return r;
821 			}
822 		}
823 	}
824 
825 	return 0;
826 }
827 
828 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
829 			    union drm_amdgpu_cs *cs)
830 {
831 	struct amdgpu_ring *ring = p->job->ring;
832 	struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
833 	struct fence *fence;
834 	struct amdgpu_job *job;
835 	int r;
836 
837 	job = p->job;
838 	p->job = NULL;
839 
840 	r = amd_sched_job_init(&job->base, &ring->sched,
841 			       entity, amdgpu_job_timeout_func,
842 			       amdgpu_job_free_func,
843 			       p->filp, &fence);
844 	if (r) {
845 		amdgpu_job_free(job);
846 		return r;
847 	}
848 
849 	job->owner = p->filp;
850 	job->ctx = entity->fence_context;
851 	p->fence = fence_get(fence);
852 	cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, fence);
853 	job->uf_sequence = cs->out.handle;
854 
855 	trace_amdgpu_cs_ioctl(job);
856 	amd_sched_entity_push_job(&job->base);
857 
858 	return 0;
859 }
860 
861 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
862 {
863 	struct amdgpu_device *adev = dev->dev_private;
864 	union drm_amdgpu_cs *cs = data;
865 	struct amdgpu_cs_parser parser = {};
866 	bool reserved_buffers = false;
867 	int i, r;
868 
869 	if (!adev->accel_working)
870 		return -EBUSY;
871 
872 	parser.adev = adev;
873 	parser.filp = filp;
874 
875 	r = amdgpu_cs_parser_init(&parser, data);
876 	if (r) {
877 		DRM_ERROR("Failed to initialize parser !\n");
878 		amdgpu_cs_parser_fini(&parser, r, false);
879 		r = amdgpu_cs_handle_lockup(adev, r);
880 		return r;
881 	}
882 	r = amdgpu_cs_parser_bos(&parser, data);
883 	if (r == -ENOMEM)
884 		DRM_ERROR("Not enough memory for command submission!\n");
885 	else if (r && r != -ERESTARTSYS)
886 		DRM_ERROR("Failed to process the buffer list %d!\n", r);
887 	else if (!r) {
888 		reserved_buffers = true;
889 		r = amdgpu_cs_ib_fill(adev, &parser);
890 	}
891 
892 	if (!r) {
893 		r = amdgpu_cs_dependencies(adev, &parser);
894 		if (r)
895 			DRM_ERROR("Failed in the dependencies handling %d!\n", r);
896 	}
897 
898 	if (r)
899 		goto out;
900 
901 	for (i = 0; i < parser.job->num_ibs; i++)
902 		trace_amdgpu_cs(&parser, i);
903 
904 	r = amdgpu_cs_ib_vm_chunk(adev, &parser);
905 	if (r)
906 		goto out;
907 
908 	r = amdgpu_cs_submit(&parser, cs);
909 
910 out:
911 	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
912 	r = amdgpu_cs_handle_lockup(adev, r);
913 	return r;
914 }
915 
916 /**
917  * amdgpu_cs_wait_ioctl - wait for a command submission to finish
918  *
919  * @dev: drm device
920  * @data: data from userspace
921  * @filp: file private
922  *
923  * Wait for the command submission identified by handle to finish.
924  */
925 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
926 			 struct drm_file *filp)
927 {
928 	union drm_amdgpu_wait_cs *wait = data;
929 	struct amdgpu_device *adev = dev->dev_private;
930 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
931 	struct amdgpu_ring *ring = NULL;
932 	struct amdgpu_ctx *ctx;
933 	struct fence *fence;
934 	long r;
935 
936 	r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
937 			       wait->in.ring, &ring);
938 	if (r)
939 		return r;
940 
941 	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
942 	if (ctx == NULL)
943 		return -EINVAL;
944 
945 	fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
946 	if (IS_ERR(fence))
947 		r = PTR_ERR(fence);
948 	else if (fence) {
949 		r = fence_wait_timeout(fence, true, timeout);
950 		fence_put(fence);
951 	} else
952 		r = 1;
953 
954 	amdgpu_ctx_put(ctx);
955 	if (r < 0)
956 		return r;
957 
958 	memset(wait, 0, sizeof(*wait));
959 	wait->out.status = (r == 0);
960 
961 	return 0;
962 }
963 
964 /**
965  * amdgpu_cs_find_bo_va - find bo_va for VM address
966  *
967  * @parser: command submission parser context
968  * @addr: VM address
969  * @bo: resulting BO of the mapping found
970  *
971  * Search the buffer objects in the command submission context for a certain
972  * virtual memory address. Returns allocation structure when found, NULL
973  * otherwise.
974  */
975 struct amdgpu_bo_va_mapping *
976 amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
977 		       uint64_t addr, struct amdgpu_bo **bo)
978 {
979 	struct amdgpu_bo_va_mapping *mapping;
980 	unsigned i;
981 
982 	if (!parser->bo_list)
983 		return NULL;
984 
985 	addr /= AMDGPU_GPU_PAGE_SIZE;
986 
987 	for (i = 0; i < parser->bo_list->num_entries; i++) {
988 		struct amdgpu_bo_list_entry *lobj;
989 
990 		lobj = &parser->bo_list->array[i];
991 		if (!lobj->bo_va)
992 			continue;
993 
994 		list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
995 			if (mapping->it.start > addr ||
996 			    addr > mapping->it.last)
997 				continue;
998 
999 			*bo = lobj->bo_va->bo;
1000 			return mapping;
1001 		}
1002 
1003 		list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
1004 			if (mapping->it.start > addr ||
1005 			    addr > mapping->it.last)
1006 				continue;
1007 
1008 			*bo = lobj->bo_va->bo;
1009 			return mapping;
1010 		}
1011 	}
1012 
1013 	return NULL;
1014 }
1015