1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <drm/drm_exec.h>
26 
27 #include "amdgpu_mes.h"
28 #include "amdgpu.h"
29 #include "soc15_common.h"
30 #include "amdgpu_mes_ctx.h"
31 
32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
33 #define AMDGPU_ONE_DOORBELL_SIZE 8
34 
35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
36 {
37 	return roundup(AMDGPU_ONE_DOORBELL_SIZE *
38 		       AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,
39 		       PAGE_SIZE);
40 }
41 
42 static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
43 					 struct amdgpu_mes_process *process,
44 					 int ip_type, uint64_t *doorbell_index)
45 {
46 	unsigned int offset, found;
47 	struct amdgpu_mes *mes = &adev->mes;
48 
49 	if (ip_type == AMDGPU_RING_TYPE_SDMA)
50 		offset = adev->doorbell_index.sdma_engine[0];
51 	else
52 		offset = 0;
53 
54 	found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);
55 	if (found >= mes->num_mes_dbs) {
56 		DRM_WARN("No doorbell available\n");
57 		return -ENOSPC;
58 	}
59 
60 	set_bit(found, mes->doorbell_bitmap);
61 
62 	/* Get the absolute doorbell index on BAR */
63 	*doorbell_index = mes->db_start_dw_offset + found * 2;
64 	return 0;
65 }
66 
67 static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,
68 					   struct amdgpu_mes_process *process,
69 					   uint32_t doorbell_index)
70 {
71 	unsigned int old, rel_index;
72 	struct amdgpu_mes *mes = &adev->mes;
73 
74 	/* Find the relative index of the doorbell in this object */
75 	rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;
76 	old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);
77 	WARN_ON(!old);
78 }
79 
80 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev)
81 {
82 	int i;
83 	struct amdgpu_mes *mes = &adev->mes;
84 
85 	/* Bitmap for dynamic allocation of kernel doorbells */
86 	mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);
87 	if (!mes->doorbell_bitmap) {
88 		DRM_ERROR("Failed to allocate MES doorbell bitmap\n");
89 		return -ENOMEM;
90 	}
91 
92 	mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;
93 	for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) {
94 		adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;
95 		set_bit(i, mes->doorbell_bitmap);
96 	}
97 
98 	return 0;
99 }
100 
101 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev)
102 {
103 	bitmap_free(adev->mes.doorbell_bitmap);
104 }
105 
106 int amdgpu_mes_init(struct amdgpu_device *adev)
107 {
108 	int i, r;
109 
110 	adev->mes.adev = adev;
111 
112 	idr_init(&adev->mes.pasid_idr);
113 	idr_init(&adev->mes.gang_id_idr);
114 	idr_init(&adev->mes.queue_id_idr);
115 	ida_init(&adev->mes.doorbell_ida);
116 	spin_lock_init(&adev->mes.queue_id_lock);
117 	spin_lock_init(&adev->mes.ring_lock);
118 	mutex_init(&adev->mes.mutex_hidden);
119 
120 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
121 	adev->mes.vmid_mask_mmhub = 0xffffff00;
122 	adev->mes.vmid_mask_gfxhub = 0xffffff00;
123 
124 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
125 		/* use only 1st MEC pipes */
126 		if (i >= 4)
127 			continue;
128 		adev->mes.compute_hqd_mask[i] = 0xc;
129 	}
130 
131 	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
132 		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;
133 
134 	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
135 		if (adev->ip_versions[SDMA0_HWIP][0] < IP_VERSION(6, 0, 0))
136 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;
137 		/* zero sdma_hqd_mask for non-existent engine */
138 		else if (adev->sdma.num_instances == 1)
139 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;
140 		else
141 			adev->mes.sdma_hqd_mask[i] = 0xfc;
142 	}
143 
144 	r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs);
145 	if (r) {
146 		dev_err(adev->dev,
147 			"(%d) ring trail_fence_offs wb alloc failed\n", r);
148 		goto error_ids;
149 	}
150 	adev->mes.sch_ctx_gpu_addr =
151 		adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4);
152 	adev->mes.sch_ctx_ptr =
153 		(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs];
154 
155 	r = amdgpu_device_wb_get(adev, &adev->mes.query_status_fence_offs);
156 	if (r) {
157 		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
158 		dev_err(adev->dev,
159 			"(%d) query_status_fence_offs wb alloc failed\n", r);
160 		goto error_ids;
161 	}
162 	adev->mes.query_status_fence_gpu_addr =
163 		adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4);
164 	adev->mes.query_status_fence_ptr =
165 		(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs];
166 
167 	r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs);
168 	if (r) {
169 		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
170 		amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
171 		dev_err(adev->dev,
172 			"(%d) read_val_offs alloc failed\n", r);
173 		goto error_ids;
174 	}
175 	adev->mes.read_val_gpu_addr =
176 		adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);
177 	adev->mes.read_val_ptr =
178 		(uint32_t *)&adev->wb.wb[adev->mes.read_val_offs];
179 
180 	r = amdgpu_mes_doorbell_init(adev);
181 	if (r)
182 		goto error;
183 
184 	return 0;
185 
186 error:
187 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
188 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
189 	amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
190 error_ids:
191 	idr_destroy(&adev->mes.pasid_idr);
192 	idr_destroy(&adev->mes.gang_id_idr);
193 	idr_destroy(&adev->mes.queue_id_idr);
194 	ida_destroy(&adev->mes.doorbell_ida);
195 	mutex_destroy(&adev->mes.mutex_hidden);
196 	return r;
197 }
198 
199 void amdgpu_mes_fini(struct amdgpu_device *adev)
200 {
201 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
202 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
203 	amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
204 	amdgpu_mes_doorbell_free(adev);
205 
206 	idr_destroy(&adev->mes.pasid_idr);
207 	idr_destroy(&adev->mes.gang_id_idr);
208 	idr_destroy(&adev->mes.queue_id_idr);
209 	ida_destroy(&adev->mes.doorbell_ida);
210 	mutex_destroy(&adev->mes.mutex_hidden);
211 }
212 
213 static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q)
214 {
215 	amdgpu_bo_free_kernel(&q->mqd_obj,
216 			      &q->mqd_gpu_addr,
217 			      &q->mqd_cpu_ptr);
218 }
219 
220 int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,
221 			      struct amdgpu_vm *vm)
222 {
223 	struct amdgpu_mes_process *process;
224 	int r;
225 
226 	/* allocate the mes process buffer */
227 	process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL);
228 	if (!process) {
229 		DRM_ERROR("no more memory to create mes process\n");
230 		return -ENOMEM;
231 	}
232 
233 	/* allocate the process context bo and map it */
234 	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,
235 				    AMDGPU_GEM_DOMAIN_GTT,
236 				    &process->proc_ctx_bo,
237 				    &process->proc_ctx_gpu_addr,
238 				    &process->proc_ctx_cpu_ptr);
239 	if (r) {
240 		DRM_ERROR("failed to allocate process context bo\n");
241 		goto clean_up_memory;
242 	}
243 	memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
244 
245 	/*
246 	 * Avoid taking any other locks under MES lock to avoid circular
247 	 * lock dependencies.
248 	 */
249 	amdgpu_mes_lock(&adev->mes);
250 
251 	/* add the mes process to idr list */
252 	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,
253 		      GFP_KERNEL);
254 	if (r < 0) {
255 		DRM_ERROR("failed to lock pasid=%d\n", pasid);
256 		goto clean_up_ctx;
257 	}
258 
259 	INIT_LIST_HEAD(&process->gang_list);
260 	process->vm = vm;
261 	process->pasid = pasid;
262 	process->process_quantum = adev->mes.default_process_quantum;
263 	process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo);
264 
265 	amdgpu_mes_unlock(&adev->mes);
266 	return 0;
267 
268 clean_up_ctx:
269 	amdgpu_mes_unlock(&adev->mes);
270 	amdgpu_bo_free_kernel(&process->proc_ctx_bo,
271 			      &process->proc_ctx_gpu_addr,
272 			      &process->proc_ctx_cpu_ptr);
273 clean_up_memory:
274 	kfree(process);
275 	return r;
276 }
277 
278 void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid)
279 {
280 	struct amdgpu_mes_process *process;
281 	struct amdgpu_mes_gang *gang, *tmp1;
282 	struct amdgpu_mes_queue *queue, *tmp2;
283 	struct mes_remove_queue_input queue_input;
284 	unsigned long flags;
285 	int r;
286 
287 	/*
288 	 * Avoid taking any other locks under MES lock to avoid circular
289 	 * lock dependencies.
290 	 */
291 	amdgpu_mes_lock(&adev->mes);
292 
293 	process = idr_find(&adev->mes.pasid_idr, pasid);
294 	if (!process) {
295 		DRM_WARN("pasid %d doesn't exist\n", pasid);
296 		amdgpu_mes_unlock(&adev->mes);
297 		return;
298 	}
299 
300 	/* Remove all queues from hardware */
301 	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
302 		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
303 			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
304 			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
305 			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
306 
307 			queue_input.doorbell_offset = queue->doorbell_off;
308 			queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
309 
310 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,
311 							     &queue_input);
312 			if (r)
313 				DRM_WARN("failed to remove hardware queue\n");
314 		}
315 
316 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
317 	}
318 
319 	idr_remove(&adev->mes.pasid_idr, pasid);
320 	amdgpu_mes_unlock(&adev->mes);
321 
322 	/* free all memory allocated by the process */
323 	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
324 		/* free all queues in the gang */
325 		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
326 			amdgpu_mes_queue_free_mqd(queue);
327 			list_del(&queue->list);
328 			kfree(queue);
329 		}
330 		amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
331 				      &gang->gang_ctx_gpu_addr,
332 				      &gang->gang_ctx_cpu_ptr);
333 		list_del(&gang->list);
334 		kfree(gang);
335 
336 	}
337 	amdgpu_bo_free_kernel(&process->proc_ctx_bo,
338 			      &process->proc_ctx_gpu_addr,
339 			      &process->proc_ctx_cpu_ptr);
340 	kfree(process);
341 }
342 
343 int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,
344 			struct amdgpu_mes_gang_properties *gprops,
345 			int *gang_id)
346 {
347 	struct amdgpu_mes_process *process;
348 	struct amdgpu_mes_gang *gang;
349 	int r;
350 
351 	/* allocate the mes gang buffer */
352 	gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL);
353 	if (!gang) {
354 		return -ENOMEM;
355 	}
356 
357 	/* allocate the gang context bo and map it to cpu space */
358 	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,
359 				    AMDGPU_GEM_DOMAIN_GTT,
360 				    &gang->gang_ctx_bo,
361 				    &gang->gang_ctx_gpu_addr,
362 				    &gang->gang_ctx_cpu_ptr);
363 	if (r) {
364 		DRM_ERROR("failed to allocate process context bo\n");
365 		goto clean_up_mem;
366 	}
367 	memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
368 
369 	/*
370 	 * Avoid taking any other locks under MES lock to avoid circular
371 	 * lock dependencies.
372 	 */
373 	amdgpu_mes_lock(&adev->mes);
374 
375 	process = idr_find(&adev->mes.pasid_idr, pasid);
376 	if (!process) {
377 		DRM_ERROR("pasid %d doesn't exist\n", pasid);
378 		r = -EINVAL;
379 		goto clean_up_ctx;
380 	}
381 
382 	/* add the mes gang to idr list */
383 	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,
384 		      GFP_KERNEL);
385 	if (r < 0) {
386 		DRM_ERROR("failed to allocate idr for gang\n");
387 		goto clean_up_ctx;
388 	}
389 
390 	gang->gang_id = r;
391 	*gang_id = r;
392 
393 	INIT_LIST_HEAD(&gang->queue_list);
394 	gang->process = process;
395 	gang->priority = gprops->priority;
396 	gang->gang_quantum = gprops->gang_quantum ?
397 		gprops->gang_quantum : adev->mes.default_gang_quantum;
398 	gang->global_priority_level = gprops->global_priority_level;
399 	gang->inprocess_gang_priority = gprops->inprocess_gang_priority;
400 	list_add_tail(&gang->list, &process->gang_list);
401 
402 	amdgpu_mes_unlock(&adev->mes);
403 	return 0;
404 
405 clean_up_ctx:
406 	amdgpu_mes_unlock(&adev->mes);
407 	amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
408 			      &gang->gang_ctx_gpu_addr,
409 			      &gang->gang_ctx_cpu_ptr);
410 clean_up_mem:
411 	kfree(gang);
412 	return r;
413 }
414 
415 int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id)
416 {
417 	struct amdgpu_mes_gang *gang;
418 
419 	/*
420 	 * Avoid taking any other locks under MES lock to avoid circular
421 	 * lock dependencies.
422 	 */
423 	amdgpu_mes_lock(&adev->mes);
424 
425 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);
426 	if (!gang) {
427 		DRM_ERROR("gang id %d doesn't exist\n", gang_id);
428 		amdgpu_mes_unlock(&adev->mes);
429 		return -EINVAL;
430 	}
431 
432 	if (!list_empty(&gang->queue_list)) {
433 		DRM_ERROR("queue list is not empty\n");
434 		amdgpu_mes_unlock(&adev->mes);
435 		return -EBUSY;
436 	}
437 
438 	idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
439 	list_del(&gang->list);
440 	amdgpu_mes_unlock(&adev->mes);
441 
442 	amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
443 			      &gang->gang_ctx_gpu_addr,
444 			      &gang->gang_ctx_cpu_ptr);
445 
446 	kfree(gang);
447 
448 	return 0;
449 }
450 
451 int amdgpu_mes_suspend(struct amdgpu_device *adev)
452 {
453 	struct idr *idp;
454 	struct amdgpu_mes_process *process;
455 	struct amdgpu_mes_gang *gang;
456 	struct mes_suspend_gang_input input;
457 	int r, pasid;
458 
459 	/*
460 	 * Avoid taking any other locks under MES lock to avoid circular
461 	 * lock dependencies.
462 	 */
463 	amdgpu_mes_lock(&adev->mes);
464 
465 	idp = &adev->mes.pasid_idr;
466 
467 	idr_for_each_entry(idp, process, pasid) {
468 		list_for_each_entry(gang, &process->gang_list, list) {
469 			r = adev->mes.funcs->suspend_gang(&adev->mes, &input);
470 			if (r)
471 				DRM_ERROR("failed to suspend pasid %d gangid %d",
472 					 pasid, gang->gang_id);
473 		}
474 	}
475 
476 	amdgpu_mes_unlock(&adev->mes);
477 	return 0;
478 }
479 
480 int amdgpu_mes_resume(struct amdgpu_device *adev)
481 {
482 	struct idr *idp;
483 	struct amdgpu_mes_process *process;
484 	struct amdgpu_mes_gang *gang;
485 	struct mes_resume_gang_input input;
486 	int r, pasid;
487 
488 	/*
489 	 * Avoid taking any other locks under MES lock to avoid circular
490 	 * lock dependencies.
491 	 */
492 	amdgpu_mes_lock(&adev->mes);
493 
494 	idp = &adev->mes.pasid_idr;
495 
496 	idr_for_each_entry(idp, process, pasid) {
497 		list_for_each_entry(gang, &process->gang_list, list) {
498 			r = adev->mes.funcs->resume_gang(&adev->mes, &input);
499 			if (r)
500 				DRM_ERROR("failed to resume pasid %d gangid %d",
501 					 pasid, gang->gang_id);
502 		}
503 	}
504 
505 	amdgpu_mes_unlock(&adev->mes);
506 	return 0;
507 }
508 
509 static int amdgpu_mes_queue_alloc_mqd(struct amdgpu_device *adev,
510 				     struct amdgpu_mes_queue *q,
511 				     struct amdgpu_mes_queue_properties *p)
512 {
513 	struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
514 	u32 mqd_size = mqd_mgr->mqd_size;
515 	int r;
516 
517 	r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
518 				    AMDGPU_GEM_DOMAIN_GTT,
519 				    &q->mqd_obj,
520 				    &q->mqd_gpu_addr, &q->mqd_cpu_ptr);
521 	if (r) {
522 		dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r);
523 		return r;
524 	}
525 	memset(q->mqd_cpu_ptr, 0, mqd_size);
526 
527 	r = amdgpu_bo_reserve(q->mqd_obj, false);
528 	if (unlikely(r != 0))
529 		goto clean_up;
530 
531 	return 0;
532 
533 clean_up:
534 	amdgpu_bo_free_kernel(&q->mqd_obj,
535 			      &q->mqd_gpu_addr,
536 			      &q->mqd_cpu_ptr);
537 	return r;
538 }
539 
540 static void amdgpu_mes_queue_init_mqd(struct amdgpu_device *adev,
541 				     struct amdgpu_mes_queue *q,
542 				     struct amdgpu_mes_queue_properties *p)
543 {
544 	struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
545 	struct amdgpu_mqd_prop mqd_prop = {0};
546 
547 	mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr;
548 	mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr;
549 	mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr;
550 	mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr;
551 	mqd_prop.queue_size = p->queue_size;
552 	mqd_prop.use_doorbell = true;
553 	mqd_prop.doorbell_index = p->doorbell_off;
554 	mqd_prop.eop_gpu_addr = p->eop_gpu_addr;
555 	mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority;
556 	mqd_prop.hqd_queue_priority = p->hqd_queue_priority;
557 	mqd_prop.hqd_active = false;
558 
559 	if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
560 	    p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
561 		mutex_lock(&adev->srbm_mutex);
562 		amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0);
563 	}
564 
565 	mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop);
566 
567 	if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
568 	    p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
569 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
570 		mutex_unlock(&adev->srbm_mutex);
571 	}
572 
573 	amdgpu_bo_unreserve(q->mqd_obj);
574 }
575 
576 int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
577 			    struct amdgpu_mes_queue_properties *qprops,
578 			    int *queue_id)
579 {
580 	struct amdgpu_mes_queue *queue;
581 	struct amdgpu_mes_gang *gang;
582 	struct mes_add_queue_input queue_input;
583 	unsigned long flags;
584 	int r;
585 
586 	memset(&queue_input, 0, sizeof(struct mes_add_queue_input));
587 
588 	/* allocate the mes queue buffer */
589 	queue = kzalloc(sizeof(struct amdgpu_mes_queue), GFP_KERNEL);
590 	if (!queue) {
591 		DRM_ERROR("Failed to allocate memory for queue\n");
592 		return -ENOMEM;
593 	}
594 
595 	/* Allocate the queue mqd */
596 	r = amdgpu_mes_queue_alloc_mqd(adev, queue, qprops);
597 	if (r)
598 		goto clean_up_memory;
599 
600 	/*
601 	 * Avoid taking any other locks under MES lock to avoid circular
602 	 * lock dependencies.
603 	 */
604 	amdgpu_mes_lock(&adev->mes);
605 
606 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);
607 	if (!gang) {
608 		DRM_ERROR("gang id %d doesn't exist\n", gang_id);
609 		r = -EINVAL;
610 		goto clean_up_mqd;
611 	}
612 
613 	/* add the mes gang to idr list */
614 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
615 	r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,
616 		      GFP_ATOMIC);
617 	if (r < 0) {
618 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
619 		goto clean_up_mqd;
620 	}
621 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
622 	*queue_id = queue->queue_id = r;
623 
624 	/* allocate a doorbell index for the queue */
625 	r = amdgpu_mes_kernel_doorbell_get(adev, gang->process,
626 					  qprops->queue_type,
627 					  &qprops->doorbell_off);
628 	if (r)
629 		goto clean_up_queue_id;
630 
631 	/* initialize the queue mqd */
632 	amdgpu_mes_queue_init_mqd(adev, queue, qprops);
633 
634 	/* add hw queue to mes */
635 	queue_input.process_id = gang->process->pasid;
636 
637 	queue_input.page_table_base_addr =
638 		adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr -
639 		adev->gmc.vram_start;
640 
641 	queue_input.process_va_start = 0;
642 	queue_input.process_va_end =
643 		(adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
644 	queue_input.process_quantum = gang->process->process_quantum;
645 	queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr;
646 	queue_input.gang_quantum = gang->gang_quantum;
647 	queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
648 	queue_input.inprocess_gang_priority = gang->inprocess_gang_priority;
649 	queue_input.gang_global_priority_level = gang->global_priority_level;
650 	queue_input.doorbell_offset = qprops->doorbell_off;
651 	queue_input.mqd_addr = queue->mqd_gpu_addr;
652 	queue_input.wptr_addr = qprops->wptr_gpu_addr;
653 	queue_input.wptr_mc_addr = qprops->wptr_mc_addr;
654 	queue_input.queue_type = qprops->queue_type;
655 	queue_input.paging = qprops->paging;
656 	queue_input.is_kfd_process = 0;
657 
658 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
659 	if (r) {
660 		DRM_ERROR("failed to add hardware queue to MES, doorbell=0x%llx\n",
661 			  qprops->doorbell_off);
662 		goto clean_up_doorbell;
663 	}
664 
665 	DRM_DEBUG("MES hw queue was added, pasid=%d, gang id=%d, "
666 		  "queue type=%d, doorbell=0x%llx\n",
667 		  gang->process->pasid, gang_id, qprops->queue_type,
668 		  qprops->doorbell_off);
669 
670 	queue->ring = qprops->ring;
671 	queue->doorbell_off = qprops->doorbell_off;
672 	queue->wptr_gpu_addr = qprops->wptr_gpu_addr;
673 	queue->queue_type = qprops->queue_type;
674 	queue->paging = qprops->paging;
675 	queue->gang = gang;
676 	queue->ring->mqd_ptr = queue->mqd_cpu_ptr;
677 	list_add_tail(&queue->list, &gang->queue_list);
678 
679 	amdgpu_mes_unlock(&adev->mes);
680 	return 0;
681 
682 clean_up_doorbell:
683 	amdgpu_mes_kernel_doorbell_free(adev, gang->process,
684 				       qprops->doorbell_off);
685 clean_up_queue_id:
686 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
687 	idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
688 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
689 clean_up_mqd:
690 	amdgpu_mes_unlock(&adev->mes);
691 	amdgpu_mes_queue_free_mqd(queue);
692 clean_up_memory:
693 	kfree(queue);
694 	return r;
695 }
696 
697 int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
698 {
699 	unsigned long flags;
700 	struct amdgpu_mes_queue *queue;
701 	struct amdgpu_mes_gang *gang;
702 	struct mes_remove_queue_input queue_input;
703 	int r;
704 
705 	/*
706 	 * Avoid taking any other locks under MES lock to avoid circular
707 	 * lock dependencies.
708 	 */
709 	amdgpu_mes_lock(&adev->mes);
710 
711 	/* remove the mes gang from idr list */
712 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
713 
714 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);
715 	if (!queue) {
716 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
717 		amdgpu_mes_unlock(&adev->mes);
718 		DRM_ERROR("queue id %d doesn't exist\n", queue_id);
719 		return -EINVAL;
720 	}
721 
722 	idr_remove(&adev->mes.queue_id_idr, queue_id);
723 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
724 
725 	DRM_DEBUG("try to remove queue, doorbell off = 0x%llx\n",
726 		  queue->doorbell_off);
727 
728 	gang = queue->gang;
729 	queue_input.doorbell_offset = queue->doorbell_off;
730 	queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
731 
732 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
733 	if (r)
734 		DRM_ERROR("failed to remove hardware queue, queue id = %d\n",
735 			  queue_id);
736 
737 	list_del(&queue->list);
738 	amdgpu_mes_kernel_doorbell_free(adev, gang->process,
739 				       queue->doorbell_off);
740 	amdgpu_mes_unlock(&adev->mes);
741 
742 	amdgpu_mes_queue_free_mqd(queue);
743 	kfree(queue);
744 	return 0;
745 }
746 
747 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
748 				  struct amdgpu_ring *ring,
749 				  enum amdgpu_unmap_queues_action action,
750 				  u64 gpu_addr, u64 seq)
751 {
752 	struct mes_unmap_legacy_queue_input queue_input;
753 	int r;
754 
755 	queue_input.action = action;
756 	queue_input.queue_type = ring->funcs->type;
757 	queue_input.doorbell_offset = ring->doorbell_index;
758 	queue_input.pipe_id = ring->pipe;
759 	queue_input.queue_id = ring->queue;
760 	queue_input.trail_fence_addr = gpu_addr;
761 	queue_input.trail_fence_data = seq;
762 
763 	r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);
764 	if (r)
765 		DRM_ERROR("failed to unmap legacy queue\n");
766 
767 	return r;
768 }
769 
770 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
771 {
772 	struct mes_misc_op_input op_input;
773 	int r, val = 0;
774 
775 	op_input.op = MES_MISC_OP_READ_REG;
776 	op_input.read_reg.reg_offset = reg;
777 	op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr;
778 
779 	if (!adev->mes.funcs->misc_op) {
780 		DRM_ERROR("mes rreg is not supported!\n");
781 		goto error;
782 	}
783 
784 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
785 	if (r)
786 		DRM_ERROR("failed to read reg (0x%x)\n", reg);
787 	else
788 		val = *(adev->mes.read_val_ptr);
789 
790 error:
791 	return val;
792 }
793 
794 int amdgpu_mes_wreg(struct amdgpu_device *adev,
795 		    uint32_t reg, uint32_t val)
796 {
797 	struct mes_misc_op_input op_input;
798 	int r;
799 
800 	op_input.op = MES_MISC_OP_WRITE_REG;
801 	op_input.write_reg.reg_offset = reg;
802 	op_input.write_reg.reg_value = val;
803 
804 	if (!adev->mes.funcs->misc_op) {
805 		DRM_ERROR("mes wreg is not supported!\n");
806 		r = -EINVAL;
807 		goto error;
808 	}
809 
810 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
811 	if (r)
812 		DRM_ERROR("failed to write reg (0x%x)\n", reg);
813 
814 error:
815 	return r;
816 }
817 
818 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
819 				  uint32_t reg0, uint32_t reg1,
820 				  uint32_t ref, uint32_t mask)
821 {
822 	struct mes_misc_op_input op_input;
823 	int r;
824 
825 	op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT;
826 	op_input.wrm_reg.reg0 = reg0;
827 	op_input.wrm_reg.reg1 = reg1;
828 	op_input.wrm_reg.ref = ref;
829 	op_input.wrm_reg.mask = mask;
830 
831 	if (!adev->mes.funcs->misc_op) {
832 		DRM_ERROR("mes reg_write_reg_wait is not supported!\n");
833 		r = -EINVAL;
834 		goto error;
835 	}
836 
837 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
838 	if (r)
839 		DRM_ERROR("failed to reg_write_reg_wait\n");
840 
841 error:
842 	return r;
843 }
844 
845 int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg,
846 			uint32_t val, uint32_t mask)
847 {
848 	struct mes_misc_op_input op_input;
849 	int r;
850 
851 	op_input.op = MES_MISC_OP_WRM_REG_WAIT;
852 	op_input.wrm_reg.reg0 = reg;
853 	op_input.wrm_reg.ref = val;
854 	op_input.wrm_reg.mask = mask;
855 
856 	if (!adev->mes.funcs->misc_op) {
857 		DRM_ERROR("mes reg wait is not supported!\n");
858 		r = -EINVAL;
859 		goto error;
860 	}
861 
862 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
863 	if (r)
864 		DRM_ERROR("failed to reg_write_reg_wait\n");
865 
866 error:
867 	return r;
868 }
869 
870 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
871 				uint64_t process_context_addr,
872 				uint32_t spi_gdbg_per_vmid_cntl,
873 				const uint32_t *tcp_watch_cntl,
874 				uint32_t flags,
875 				bool trap_en)
876 {
877 	struct mes_misc_op_input op_input = {0};
878 	int r;
879 
880 	if (!adev->mes.funcs->misc_op) {
881 		DRM_ERROR("mes set shader debugger is not supported!\n");
882 		return -EINVAL;
883 	}
884 
885 	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
886 	op_input.set_shader_debugger.process_context_addr = process_context_addr;
887 	op_input.set_shader_debugger.flags.u32all = flags;
888 
889 	/* use amdgpu mes_flush_shader_debugger instead */
890 	if (op_input.set_shader_debugger.flags.process_ctx_flush)
891 		return -EINVAL;
892 
893 	op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl;
894 	memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl,
895 			sizeof(op_input.set_shader_debugger.tcp_watch_cntl));
896 
897 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
898 			AMDGPU_MES_API_VERSION_SHIFT) >= 14)
899 		op_input.set_shader_debugger.trap_en = trap_en;
900 
901 	amdgpu_mes_lock(&adev->mes);
902 
903 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
904 	if (r)
905 		DRM_ERROR("failed to set_shader_debugger\n");
906 
907 	amdgpu_mes_unlock(&adev->mes);
908 
909 	return r;
910 }
911 
912 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
913 				     uint64_t process_context_addr)
914 {
915 	struct mes_misc_op_input op_input = {0};
916 	int r;
917 
918 	if (!adev->mes.funcs->misc_op) {
919 		DRM_ERROR("mes flush shader debugger is not supported!\n");
920 		return -EINVAL;
921 	}
922 
923 	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
924 	op_input.set_shader_debugger.process_context_addr = process_context_addr;
925 	op_input.set_shader_debugger.flags.process_ctx_flush = true;
926 
927 	amdgpu_mes_lock(&adev->mes);
928 
929 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
930 	if (r)
931 		DRM_ERROR("failed to set_shader_debugger\n");
932 
933 	amdgpu_mes_unlock(&adev->mes);
934 
935 	return r;
936 }
937 
938 static void
939 amdgpu_mes_ring_to_queue_props(struct amdgpu_device *adev,
940 			       struct amdgpu_ring *ring,
941 			       struct amdgpu_mes_queue_properties *props)
942 {
943 	props->queue_type = ring->funcs->type;
944 	props->hqd_base_gpu_addr = ring->gpu_addr;
945 	props->rptr_gpu_addr = ring->rptr_gpu_addr;
946 	props->wptr_gpu_addr = ring->wptr_gpu_addr;
947 	props->wptr_mc_addr =
948 		ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs;
949 	props->queue_size = ring->ring_size;
950 	props->eop_gpu_addr = ring->eop_gpu_addr;
951 	props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
952 	props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
953 	props->paging = false;
954 	props->ring = ring;
955 }
956 
957 #define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng)			\
958 do {									\
959        if (id_offs < AMDGPU_MES_CTX_MAX_OFFS)				\
960 		return offsetof(struct amdgpu_mes_ctx_meta_data,	\
961 				_eng[ring->idx].slots[id_offs]);        \
962        else if (id_offs == AMDGPU_MES_CTX_RING_OFFS)			\
963 		return offsetof(struct amdgpu_mes_ctx_meta_data,        \
964 				_eng[ring->idx].ring);                  \
965        else if (id_offs == AMDGPU_MES_CTX_IB_OFFS)			\
966 		return offsetof(struct amdgpu_mes_ctx_meta_data,        \
967 				_eng[ring->idx].ib);                    \
968        else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS)			\
969 		return offsetof(struct amdgpu_mes_ctx_meta_data,        \
970 				_eng[ring->idx].padding);               \
971 } while(0)
972 
973 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs)
974 {
975 	switch (ring->funcs->type) {
976 	case AMDGPU_RING_TYPE_GFX:
977 		DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx);
978 		break;
979 	case AMDGPU_RING_TYPE_COMPUTE:
980 		DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute);
981 		break;
982 	case AMDGPU_RING_TYPE_SDMA:
983 		DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma);
984 		break;
985 	default:
986 		break;
987 	}
988 
989 	WARN_ON(1);
990 	return -EINVAL;
991 }
992 
993 int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id,
994 			int queue_type, int idx,
995 			struct amdgpu_mes_ctx_data *ctx_data,
996 			struct amdgpu_ring **out)
997 {
998 	struct amdgpu_ring *ring;
999 	struct amdgpu_mes_gang *gang;
1000 	struct amdgpu_mes_queue_properties qprops = {0};
1001 	int r, queue_id, pasid;
1002 
1003 	/*
1004 	 * Avoid taking any other locks under MES lock to avoid circular
1005 	 * lock dependencies.
1006 	 */
1007 	amdgpu_mes_lock(&adev->mes);
1008 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);
1009 	if (!gang) {
1010 		DRM_ERROR("gang id %d doesn't exist\n", gang_id);
1011 		amdgpu_mes_unlock(&adev->mes);
1012 		return -EINVAL;
1013 	}
1014 	pasid = gang->process->pasid;
1015 
1016 	ring = kzalloc(sizeof(struct amdgpu_ring), GFP_KERNEL);
1017 	if (!ring) {
1018 		amdgpu_mes_unlock(&adev->mes);
1019 		return -ENOMEM;
1020 	}
1021 
1022 	ring->ring_obj = NULL;
1023 	ring->use_doorbell = true;
1024 	ring->is_mes_queue = true;
1025 	ring->mes_ctx = ctx_data;
1026 	ring->idx = idx;
1027 	ring->no_scheduler = true;
1028 
1029 	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1030 		int offset = offsetof(struct amdgpu_mes_ctx_meta_data,
1031 				      compute[ring->idx].mec_hpd);
1032 		ring->eop_gpu_addr =
1033 			amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
1034 	}
1035 
1036 	switch (queue_type) {
1037 	case AMDGPU_RING_TYPE_GFX:
1038 		ring->funcs = adev->gfx.gfx_ring[0].funcs;
1039 		ring->me = adev->gfx.gfx_ring[0].me;
1040 		ring->pipe = adev->gfx.gfx_ring[0].pipe;
1041 		break;
1042 	case AMDGPU_RING_TYPE_COMPUTE:
1043 		ring->funcs = adev->gfx.compute_ring[0].funcs;
1044 		ring->me = adev->gfx.compute_ring[0].me;
1045 		ring->pipe = adev->gfx.compute_ring[0].pipe;
1046 		break;
1047 	case AMDGPU_RING_TYPE_SDMA:
1048 		ring->funcs = adev->sdma.instance[0].ring.funcs;
1049 		break;
1050 	default:
1051 		BUG();
1052 	}
1053 
1054 	r = amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1055 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
1056 	if (r)
1057 		goto clean_up_memory;
1058 
1059 	amdgpu_mes_ring_to_queue_props(adev, ring, &qprops);
1060 
1061 	dma_fence_wait(gang->process->vm->last_update, false);
1062 	dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false);
1063 	amdgpu_mes_unlock(&adev->mes);
1064 
1065 	r = amdgpu_mes_add_hw_queue(adev, gang_id, &qprops, &queue_id);
1066 	if (r)
1067 		goto clean_up_ring;
1068 
1069 	ring->hw_queue_id = queue_id;
1070 	ring->doorbell_index = qprops.doorbell_off;
1071 
1072 	if (queue_type == AMDGPU_RING_TYPE_GFX)
1073 		sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id);
1074 	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
1075 		sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id,
1076 			queue_id);
1077 	else if (queue_type == AMDGPU_RING_TYPE_SDMA)
1078 		sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id,
1079 			queue_id);
1080 	else
1081 		BUG();
1082 
1083 	*out = ring;
1084 	return 0;
1085 
1086 clean_up_ring:
1087 	amdgpu_ring_fini(ring);
1088 clean_up_memory:
1089 	kfree(ring);
1090 	amdgpu_mes_unlock(&adev->mes);
1091 	return r;
1092 }
1093 
1094 void amdgpu_mes_remove_ring(struct amdgpu_device *adev,
1095 			    struct amdgpu_ring *ring)
1096 {
1097 	if (!ring)
1098 		return;
1099 
1100 	amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id);
1101 	del_timer_sync(&ring->fence_drv.fallback_timer);
1102 	amdgpu_ring_fini(ring);
1103 	kfree(ring);
1104 }
1105 
1106 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
1107 						   enum amdgpu_mes_priority_level prio)
1108 {
1109 	return adev->mes.aggregated_doorbells[prio];
1110 }
1111 
1112 int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev,
1113 				   struct amdgpu_mes_ctx_data *ctx_data)
1114 {
1115 	int r;
1116 
1117 	r = amdgpu_bo_create_kernel(adev,
1118 			    sizeof(struct amdgpu_mes_ctx_meta_data),
1119 			    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1120 			    &ctx_data->meta_data_obj,
1121 			    &ctx_data->meta_data_mc_addr,
1122 			    &ctx_data->meta_data_ptr);
1123 	if (r) {
1124 		dev_warn(adev->dev, "(%d) create CTX bo failed\n", r);
1125 		return r;
1126 	}
1127 
1128 	if (!ctx_data->meta_data_obj)
1129 		return -ENOMEM;
1130 
1131 	memset(ctx_data->meta_data_ptr, 0,
1132 	       sizeof(struct amdgpu_mes_ctx_meta_data));
1133 
1134 	return 0;
1135 }
1136 
1137 void amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data *ctx_data)
1138 {
1139 	if (ctx_data->meta_data_obj)
1140 		amdgpu_bo_free_kernel(&ctx_data->meta_data_obj,
1141 				      &ctx_data->meta_data_mc_addr,
1142 				      &ctx_data->meta_data_ptr);
1143 }
1144 
1145 int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
1146 				 struct amdgpu_vm *vm,
1147 				 struct amdgpu_mes_ctx_data *ctx_data)
1148 {
1149 	struct amdgpu_bo_va *bo_va;
1150 	struct amdgpu_sync sync;
1151 	struct drm_exec exec;
1152 	int r;
1153 
1154 	amdgpu_sync_create(&sync);
1155 
1156 	drm_exec_init(&exec, 0);
1157 	drm_exec_until_all_locked(&exec) {
1158 		r = drm_exec_lock_obj(&exec,
1159 				      &ctx_data->meta_data_obj->tbo.base);
1160 		drm_exec_retry_on_contention(&exec);
1161 		if (unlikely(r))
1162 			goto error_fini_exec;
1163 
1164 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
1165 		drm_exec_retry_on_contention(&exec);
1166 		if (unlikely(r))
1167 			goto error_fini_exec;
1168 	}
1169 
1170 	bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj);
1171 	if (!bo_va) {
1172 		DRM_ERROR("failed to create bo_va for meta data BO\n");
1173 		r = -ENOMEM;
1174 		goto error_fini_exec;
1175 	}
1176 
1177 	r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0,
1178 			     sizeof(struct amdgpu_mes_ctx_meta_data),
1179 			     AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
1180 			     AMDGPU_PTE_EXECUTABLE);
1181 
1182 	if (r) {
1183 		DRM_ERROR("failed to do bo_map on meta data, err=%d\n", r);
1184 		goto error_del_bo_va;
1185 	}
1186 
1187 	r = amdgpu_vm_bo_update(adev, bo_va, false);
1188 	if (r) {
1189 		DRM_ERROR("failed to do vm_bo_update on meta data\n");
1190 		goto error_del_bo_va;
1191 	}
1192 	amdgpu_sync_fence(&sync, bo_va->last_pt_update);
1193 
1194 	r = amdgpu_vm_update_pdes(adev, vm, false);
1195 	if (r) {
1196 		DRM_ERROR("failed to update pdes on meta data\n");
1197 		goto error_del_bo_va;
1198 	}
1199 	amdgpu_sync_fence(&sync, vm->last_update);
1200 
1201 	amdgpu_sync_wait(&sync, false);
1202 	drm_exec_fini(&exec);
1203 
1204 	amdgpu_sync_free(&sync);
1205 	ctx_data->meta_data_va = bo_va;
1206 	return 0;
1207 
1208 error_del_bo_va:
1209 	amdgpu_vm_bo_del(adev, bo_va);
1210 
1211 error_fini_exec:
1212 	drm_exec_fini(&exec);
1213 	amdgpu_sync_free(&sync);
1214 	return r;
1215 }
1216 
1217 int amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device *adev,
1218 				   struct amdgpu_mes_ctx_data *ctx_data)
1219 {
1220 	struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va;
1221 	struct amdgpu_bo *bo = ctx_data->meta_data_obj;
1222 	struct amdgpu_vm *vm = bo_va->base.vm;
1223 	struct dma_fence *fence;
1224 	struct drm_exec exec;
1225 	long r;
1226 
1227 	drm_exec_init(&exec, 0);
1228 	drm_exec_until_all_locked(&exec) {
1229 		r = drm_exec_lock_obj(&exec,
1230 				      &ctx_data->meta_data_obj->tbo.base);
1231 		drm_exec_retry_on_contention(&exec);
1232 		if (unlikely(r))
1233 			goto out_unlock;
1234 
1235 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
1236 		drm_exec_retry_on_contention(&exec);
1237 		if (unlikely(r))
1238 			goto out_unlock;
1239 	}
1240 
1241 	amdgpu_vm_bo_del(adev, bo_va);
1242 	if (!amdgpu_vm_ready(vm))
1243 		goto out_unlock;
1244 
1245 	r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP,
1246 				   &fence);
1247 	if (r)
1248 		goto out_unlock;
1249 	if (fence) {
1250 		amdgpu_bo_fence(bo, fence, true);
1251 		fence = NULL;
1252 	}
1253 
1254 	r = amdgpu_vm_clear_freed(adev, vm, &fence);
1255 	if (r || !fence)
1256 		goto out_unlock;
1257 
1258 	dma_fence_wait(fence, false);
1259 	amdgpu_bo_fence(bo, fence, true);
1260 	dma_fence_put(fence);
1261 
1262 out_unlock:
1263 	if (unlikely(r < 0))
1264 		dev_err(adev->dev, "failed to clear page tables (%ld)\n", r);
1265 	drm_exec_fini(&exec);
1266 
1267 	return r;
1268 }
1269 
1270 static int amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device *adev,
1271 					  int pasid, int *gang_id,
1272 					  int queue_type, int num_queue,
1273 					  struct amdgpu_ring **added_rings,
1274 					  struct amdgpu_mes_ctx_data *ctx_data)
1275 {
1276 	struct amdgpu_ring *ring;
1277 	struct amdgpu_mes_gang_properties gprops = {0};
1278 	int r, j;
1279 
1280 	/* create a gang for the process */
1281 	gprops.priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1282 	gprops.gang_quantum = adev->mes.default_gang_quantum;
1283 	gprops.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1284 	gprops.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1285 	gprops.global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1286 
1287 	r = amdgpu_mes_add_gang(adev, pasid, &gprops, gang_id);
1288 	if (r) {
1289 		DRM_ERROR("failed to add gang\n");
1290 		return r;
1291 	}
1292 
1293 	/* create queues for the gang */
1294 	for (j = 0; j < num_queue; j++) {
1295 		r = amdgpu_mes_add_ring(adev, *gang_id, queue_type, j,
1296 					ctx_data, &ring);
1297 		if (r) {
1298 			DRM_ERROR("failed to add ring\n");
1299 			break;
1300 		}
1301 
1302 		DRM_INFO("ring %s was added\n", ring->name);
1303 		added_rings[j] = ring;
1304 	}
1305 
1306 	return 0;
1307 }
1308 
1309 static int amdgpu_mes_test_queues(struct amdgpu_ring **added_rings)
1310 {
1311 	struct amdgpu_ring *ring;
1312 	int i, r;
1313 
1314 	for (i = 0; i < AMDGPU_MES_CTX_MAX_RINGS; i++) {
1315 		ring = added_rings[i];
1316 		if (!ring)
1317 			continue;
1318 
1319 		r = amdgpu_ring_test_helper(ring);
1320 		if (r)
1321 			return r;
1322 
1323 		r = amdgpu_ring_test_ib(ring, 1000 * 10);
1324 		if (r) {
1325 			DRM_DEV_ERROR(ring->adev->dev,
1326 				      "ring %s ib test failed (%d)\n",
1327 				      ring->name, r);
1328 			return r;
1329 		} else
1330 			DRM_INFO("ring %s ib test pass\n", ring->name);
1331 	}
1332 
1333 	return 0;
1334 }
1335 
1336 int amdgpu_mes_self_test(struct amdgpu_device *adev)
1337 {
1338 	struct amdgpu_vm *vm = NULL;
1339 	struct amdgpu_mes_ctx_data ctx_data = {0};
1340 	struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL };
1341 	int gang_ids[3] = {0};
1342 	int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 },
1343 				 { AMDGPU_RING_TYPE_COMPUTE, 1 },
1344 				 { AMDGPU_RING_TYPE_SDMA, 1} };
1345 	int i, r, pasid, k = 0;
1346 
1347 	pasid = amdgpu_pasid_alloc(16);
1348 	if (pasid < 0) {
1349 		dev_warn(adev->dev, "No more PASIDs available!");
1350 		pasid = 0;
1351 	}
1352 
1353 	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1354 	if (!vm) {
1355 		r = -ENOMEM;
1356 		goto error_pasid;
1357 	}
1358 
1359 	r = amdgpu_vm_init(adev, vm, -1);
1360 	if (r) {
1361 		DRM_ERROR("failed to initialize vm\n");
1362 		goto error_pasid;
1363 	}
1364 
1365 	r = amdgpu_mes_ctx_alloc_meta_data(adev, &ctx_data);
1366 	if (r) {
1367 		DRM_ERROR("failed to alloc ctx meta data\n");
1368 		goto error_fini;
1369 	}
1370 
1371 	ctx_data.meta_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE;
1372 	r = amdgpu_mes_ctx_map_meta_data(adev, vm, &ctx_data);
1373 	if (r) {
1374 		DRM_ERROR("failed to map ctx meta data\n");
1375 		goto error_vm;
1376 	}
1377 
1378 	r = amdgpu_mes_create_process(adev, pasid, vm);
1379 	if (r) {
1380 		DRM_ERROR("failed to create MES process\n");
1381 		goto error_vm;
1382 	}
1383 
1384 	for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
1385 		/* On GFX v10.3, fw hasn't supported to map sdma queue. */
1386 		if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(10, 3, 0) &&
1387 		    adev->ip_versions[GC_HWIP][0] < IP_VERSION(11, 0, 0) &&
1388 		    queue_types[i][0] == AMDGPU_RING_TYPE_SDMA)
1389 			continue;
1390 
1391 		r = amdgpu_mes_test_create_gang_and_queues(adev, pasid,
1392 							   &gang_ids[i],
1393 							   queue_types[i][0],
1394 							   queue_types[i][1],
1395 							   &added_rings[k],
1396 							   &ctx_data);
1397 		if (r)
1398 			goto error_queues;
1399 
1400 		k += queue_types[i][1];
1401 	}
1402 
1403 	/* start ring test and ib test for MES queues */
1404 	amdgpu_mes_test_queues(added_rings);
1405 
1406 error_queues:
1407 	/* remove all queues */
1408 	for (i = 0; i < ARRAY_SIZE(added_rings); i++) {
1409 		if (!added_rings[i])
1410 			continue;
1411 		amdgpu_mes_remove_ring(adev, added_rings[i]);
1412 	}
1413 
1414 	for (i = 0; i < ARRAY_SIZE(gang_ids); i++) {
1415 		if (!gang_ids[i])
1416 			continue;
1417 		amdgpu_mes_remove_gang(adev, gang_ids[i]);
1418 	}
1419 
1420 	amdgpu_mes_destroy_process(adev, pasid);
1421 
1422 error_vm:
1423 	amdgpu_mes_ctx_unmap_meta_data(adev, &ctx_data);
1424 
1425 error_fini:
1426 	amdgpu_vm_fini(adev, vm);
1427 
1428 error_pasid:
1429 	if (pasid)
1430 		amdgpu_pasid_free(pasid);
1431 
1432 	amdgpu_mes_ctx_free_meta_data(&ctx_data);
1433 	kfree(vm);
1434 	return 0;
1435 }
1436 
1437 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
1438 {
1439 	const struct mes_firmware_header_v1_0 *mes_hdr;
1440 	struct amdgpu_firmware_info *info;
1441 	char ucode_prefix[30];
1442 	char fw_name[40];
1443 	bool need_retry = false;
1444 	int r;
1445 
1446 	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix,
1447 				       sizeof(ucode_prefix));
1448 	if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) {
1449 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1450 			 ucode_prefix,
1451 			 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1");
1452 		need_retry = true;
1453 	} else {
1454 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1455 			 ucode_prefix,
1456 			 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
1457 	}
1458 
1459 	r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name);
1460 	if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) {
1461 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin",
1462 			 ucode_prefix);
1463 		DRM_INFO("try to fall back to %s\n", fw_name);
1464 		r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],
1465 					 fw_name);
1466 	}
1467 
1468 	if (r)
1469 		goto out;
1470 
1471 	mes_hdr = (const struct mes_firmware_header_v1_0 *)
1472 		adev->mes.fw[pipe]->data;
1473 	adev->mes.uc_start_addr[pipe] =
1474 		le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
1475 		((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
1476 	adev->mes.data_start_addr[pipe] =
1477 		le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
1478 		((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
1479 
1480 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1481 		int ucode, ucode_data;
1482 
1483 		if (pipe == AMDGPU_MES_SCHED_PIPE) {
1484 			ucode = AMDGPU_UCODE_ID_CP_MES;
1485 			ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
1486 		} else {
1487 			ucode = AMDGPU_UCODE_ID_CP_MES1;
1488 			ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
1489 		}
1490 
1491 		info = &adev->firmware.ucode[ucode];
1492 		info->ucode_id = ucode;
1493 		info->fw = adev->mes.fw[pipe];
1494 		adev->firmware.fw_size +=
1495 			ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
1496 			      PAGE_SIZE);
1497 
1498 		info = &adev->firmware.ucode[ucode_data];
1499 		info->ucode_id = ucode_data;
1500 		info->fw = adev->mes.fw[pipe];
1501 		adev->firmware.fw_size +=
1502 			ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
1503 			      PAGE_SIZE);
1504 	}
1505 
1506 	return 0;
1507 out:
1508 	amdgpu_ucode_release(&adev->mes.fw[pipe]);
1509 	return r;
1510 }
1511