xref: /openbmc/linux/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c (revision 360823a09426347ea8f232b0b0b5156d0aed0302)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/slab.h>
26 #include <linux/list.h>
27 #include "kfd_device_queue_manager.h"
28 #include "kfd_priv.h"
29 #include "kfd_kernel_queue.h"
30 #include "amdgpu_amdkfd.h"
31 #include "amdgpu_reset.h"
32 
get_queue_by_qid(struct process_queue_manager * pqm,unsigned int qid)33 static inline struct process_queue_node *get_queue_by_qid(
34 			struct process_queue_manager *pqm, unsigned int qid)
35 {
36 	struct process_queue_node *pqn;
37 
38 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
39 		if ((pqn->q && pqn->q->properties.queue_id == qid) ||
40 		    (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
41 			return pqn;
42 	}
43 
44 	return NULL;
45 }
46 
assign_queue_slot_by_qid(struct process_queue_manager * pqm,unsigned int qid)47 static int assign_queue_slot_by_qid(struct process_queue_manager *pqm,
48 				    unsigned int qid)
49 {
50 	if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
51 		return -EINVAL;
52 
53 	if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) {
54 		pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid);
55 		return -ENOSPC;
56 	}
57 
58 	return 0;
59 }
60 
find_available_queue_slot(struct process_queue_manager * pqm,unsigned int * qid)61 static int find_available_queue_slot(struct process_queue_manager *pqm,
62 					unsigned int *qid)
63 {
64 	unsigned long found;
65 
66 	found = find_first_zero_bit(pqm->queue_slot_bitmap,
67 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
68 
69 	pr_debug("The new slot id %lu\n", found);
70 
71 	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
72 		pr_info("Cannot open more queues for process with pasid 0x%x\n",
73 				pqm->process->pasid);
74 		return -ENOMEM;
75 	}
76 
77 	set_bit(found, pqm->queue_slot_bitmap);
78 	*qid = found;
79 
80 	return 0;
81 }
82 
kfd_process_dequeue_from_device(struct kfd_process_device * pdd)83 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
84 {
85 	struct kfd_node *dev = pdd->dev;
86 
87 	if (pdd->already_dequeued)
88 		return;
89 	/* The MES context flush needs to filter out the case which the
90 	 * KFD process is created without setting up the MES context and
91 	 * queue for creating a compute queue.
92 	 */
93 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
94 	if (dev->kfd->shared_resources.enable_mes && !!pdd->proc_ctx_gpu_addr &&
95 	    down_read_trylock(&dev->adev->reset_domain->sem)) {
96 		amdgpu_mes_flush_shader_debugger(dev->adev,
97 						 pdd->proc_ctx_gpu_addr);
98 		up_read(&dev->adev->reset_domain->sem);
99 	}
100 	pdd->already_dequeued = true;
101 }
102 
pqm_set_gws(struct process_queue_manager * pqm,unsigned int qid,void * gws)103 int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
104 			void *gws)
105 {
106 	struct kfd_node *dev = NULL;
107 	struct process_queue_node *pqn;
108 	struct kfd_process_device *pdd;
109 	struct kgd_mem *mem = NULL;
110 	int ret;
111 
112 	pqn = get_queue_by_qid(pqm, qid);
113 	if (!pqn) {
114 		pr_err("Queue id does not match any known queue\n");
115 		return -EINVAL;
116 	}
117 
118 	if (pqn->q)
119 		dev = pqn->q->device;
120 	if (WARN_ON(!dev))
121 		return -ENODEV;
122 
123 	pdd = kfd_get_process_device_data(dev, pqm->process);
124 	if (!pdd) {
125 		pr_err("Process device data doesn't exist\n");
126 		return -EINVAL;
127 	}
128 
129 	/* Only allow one queue per process can have GWS assigned */
130 	if (gws && pdd->qpd.num_gws)
131 		return -EBUSY;
132 
133 	if (!gws && pdd->qpd.num_gws == 0)
134 		return -EINVAL;
135 
136 	if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3) && !dev->kfd->shared_resources.enable_mes) {
137 		if (gws)
138 			ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
139 				gws, &mem);
140 		else
141 			ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
142 				pqn->q->gws);
143 		if (unlikely(ret))
144 			return ret;
145 		pqn->q->gws = mem;
146 	} else {
147 		/*
148 		 * Intentionally set GWS to a non-NULL value
149 		 * for devices that do not use GWS for global wave
150 		 * synchronization but require the formality
151 		 * of setting GWS for cooperative groups.
152 		 */
153 		pqn->q->gws = gws ? ERR_PTR(-ENOMEM) : NULL;
154 	}
155 
156 	pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;
157 
158 	return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
159 							pqn->q, NULL);
160 }
161 
kfd_process_dequeue_from_all_devices(struct kfd_process * p)162 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
163 {
164 	int i;
165 
166 	for (i = 0; i < p->n_pdds; i++)
167 		kfd_process_dequeue_from_device(p->pdds[i]);
168 }
169 
pqm_init(struct process_queue_manager * pqm,struct kfd_process * p)170 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
171 {
172 	INIT_LIST_HEAD(&pqm->queues);
173 	pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
174 					       GFP_KERNEL);
175 	if (!pqm->queue_slot_bitmap)
176 		return -ENOMEM;
177 	pqm->process = p;
178 
179 	return 0;
180 }
181 
pqm_clean_queue_resource(struct process_queue_manager * pqm,struct process_queue_node * pqn)182 static void pqm_clean_queue_resource(struct process_queue_manager *pqm,
183 				     struct process_queue_node *pqn)
184 {
185 	struct kfd_node *dev;
186 	struct kfd_process_device *pdd;
187 
188 	dev = pqn->q->device;
189 
190 	pdd = kfd_get_process_device_data(dev, pqm->process);
191 	if (!pdd) {
192 		pr_err("Process device data doesn't exist\n");
193 		return;
194 	}
195 
196 	if (pqn->q->gws) {
197 		if (KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 3) &&
198 		    !dev->kfd->shared_resources.enable_mes)
199 			amdgpu_amdkfd_remove_gws_from_process(
200 				pqm->process->kgd_process_info, pqn->q->gws);
201 		pdd->qpd.num_gws = 0;
202 	}
203 
204 	if (dev->kfd->shared_resources.enable_mes) {
205 		amdgpu_amdkfd_free_gtt_mem(dev->adev, &pqn->q->gang_ctx_bo);
206 		if (pqn->q->wptr_bo)
207 			amdgpu_amdkfd_free_gtt_mem(dev->adev, (void **)&pqn->q->wptr_bo);
208 	}
209 }
210 
pqm_uninit(struct process_queue_manager * pqm)211 void pqm_uninit(struct process_queue_manager *pqm)
212 {
213 	struct process_queue_node *pqn, *next;
214 
215 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
216 		if (pqn->q)
217 			pqm_clean_queue_resource(pqm, pqn);
218 
219 		kfd_procfs_del_queue(pqn->q);
220 		uninit_queue(pqn->q);
221 		list_del(&pqn->process_queue_list);
222 		kfree(pqn);
223 	}
224 
225 	bitmap_free(pqm->queue_slot_bitmap);
226 	pqm->queue_slot_bitmap = NULL;
227 }
228 
init_user_queue(struct process_queue_manager * pqm,struct kfd_node * dev,struct queue ** q,struct queue_properties * q_properties,struct file * f,struct amdgpu_bo * wptr_bo,unsigned int qid)229 static int init_user_queue(struct process_queue_manager *pqm,
230 				struct kfd_node *dev, struct queue **q,
231 				struct queue_properties *q_properties,
232 				struct file *f, struct amdgpu_bo *wptr_bo,
233 				unsigned int qid)
234 {
235 	int retval;
236 
237 	/* Doorbell initialized in user space*/
238 	q_properties->doorbell_ptr = NULL;
239 	q_properties->exception_status = KFD_EC_MASK(EC_QUEUE_NEW);
240 
241 	/* let DQM handle it*/
242 	q_properties->vmid = 0;
243 	q_properties->queue_id = qid;
244 
245 	retval = init_queue(q, q_properties);
246 	if (retval != 0)
247 		return retval;
248 
249 	(*q)->device = dev;
250 	(*q)->process = pqm->process;
251 
252 	if (dev->kfd->shared_resources.enable_mes) {
253 		retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,
254 						AMDGPU_MES_GANG_CTX_SIZE,
255 						&(*q)->gang_ctx_bo,
256 						&(*q)->gang_ctx_gpu_addr,
257 						&(*q)->gang_ctx_cpu_ptr,
258 						false);
259 		if (retval) {
260 			pr_err("failed to allocate gang context bo\n");
261 			goto cleanup;
262 		}
263 		memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
264 		(*q)->wptr_bo = wptr_bo;
265 	}
266 
267 	pr_debug("PQM After init queue");
268 	return 0;
269 
270 cleanup:
271 	uninit_queue(*q);
272 	*q = NULL;
273 	return retval;
274 }
275 
pqm_create_queue(struct process_queue_manager * pqm,struct kfd_node * dev,struct file * f,struct queue_properties * properties,unsigned int * qid,struct amdgpu_bo * wptr_bo,const struct kfd_criu_queue_priv_data * q_data,const void * restore_mqd,const void * restore_ctl_stack,uint32_t * p_doorbell_offset_in_process)276 int pqm_create_queue(struct process_queue_manager *pqm,
277 			    struct kfd_node *dev,
278 			    struct file *f,
279 			    struct queue_properties *properties,
280 			    unsigned int *qid,
281 			    struct amdgpu_bo *wptr_bo,
282 			    const struct kfd_criu_queue_priv_data *q_data,
283 			    const void *restore_mqd,
284 			    const void *restore_ctl_stack,
285 			    uint32_t *p_doorbell_offset_in_process)
286 {
287 	int retval;
288 	struct kfd_process_device *pdd;
289 	struct queue *q;
290 	struct process_queue_node *pqn;
291 	struct kernel_queue *kq;
292 	enum kfd_queue_type type = properties->type;
293 	unsigned int max_queues = 127; /* HWS limit */
294 
295 	/*
296 	 * On GFX 9.4.3, increase the number of queues that
297 	 * can be created to 255. No HWS limit on GFX 9.4.3.
298 	 */
299 	if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3))
300 		max_queues = 255;
301 
302 	q = NULL;
303 	kq = NULL;
304 
305 	pdd = kfd_get_process_device_data(dev, pqm->process);
306 	if (!pdd) {
307 		pr_err("Process device data doesn't exist\n");
308 		return -1;
309 	}
310 
311 	/*
312 	 * for debug process, verify that it is within the static queues limit
313 	 * currently limit is set to half of the total avail HQD slots
314 	 * If we are just about to create DIQ, the is_debug flag is not set yet
315 	 * Hence we also check the type as well
316 	 */
317 	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
318 		max_queues = dev->kfd->device_info.max_no_of_hqd/2;
319 
320 	if (pdd->qpd.queue_count >= max_queues)
321 		return -ENOSPC;
322 
323 	if (q_data) {
324 		retval = assign_queue_slot_by_qid(pqm, q_data->q_id);
325 		*qid = q_data->q_id;
326 	} else
327 		retval = find_available_queue_slot(pqm, qid);
328 
329 	if (retval != 0)
330 		return retval;
331 
332 	if (list_empty(&pdd->qpd.queues_list) &&
333 	    list_empty(&pdd->qpd.priv_queue_list))
334 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
335 
336 	pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
337 	if (!pqn) {
338 		retval = -ENOMEM;
339 		goto err_allocate_pqn;
340 	}
341 
342 	switch (type) {
343 	case KFD_QUEUE_TYPE_SDMA:
344 	case KFD_QUEUE_TYPE_SDMA_XGMI:
345 		/* SDMA queues are always allocated statically no matter
346 		 * which scheduler mode is used. We also do not need to
347 		 * check whether a SDMA queue can be allocated here, because
348 		 * allocate_sdma_queue() in create_queue() has the
349 		 * corresponding check logic.
350 		 */
351 		retval = init_user_queue(pqm, dev, &q, properties, f, wptr_bo, *qid);
352 		if (retval != 0)
353 			goto err_create_queue;
354 		pqn->q = q;
355 		pqn->kq = NULL;
356 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
357 						    restore_mqd, restore_ctl_stack);
358 		print_queue(q);
359 		break;
360 
361 	case KFD_QUEUE_TYPE_COMPUTE:
362 		/* check if there is over subscription */
363 		if ((dev->dqm->sched_policy ==
364 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
365 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
366 		(dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {
367 			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
368 			retval = -EPERM;
369 			goto err_create_queue;
370 		}
371 
372 		retval = init_user_queue(pqm, dev, &q, properties, f, wptr_bo, *qid);
373 		if (retval != 0)
374 			goto err_create_queue;
375 		pqn->q = q;
376 		pqn->kq = NULL;
377 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
378 						    restore_mqd, restore_ctl_stack);
379 		print_queue(q);
380 		break;
381 	case KFD_QUEUE_TYPE_DIQ:
382 		kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
383 		if (!kq) {
384 			retval = -ENOMEM;
385 			goto err_create_queue;
386 		}
387 		kq->queue->properties.queue_id = *qid;
388 		pqn->kq = kq;
389 		pqn->q = NULL;
390 		retval = kfd_process_drain_interrupts(pdd);
391 		if (retval)
392 			break;
393 
394 		retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
395 							kq, &pdd->qpd);
396 		break;
397 	default:
398 		WARN(1, "Invalid queue type %d", type);
399 		retval = -EINVAL;
400 	}
401 
402 	if (retval != 0) {
403 		pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n",
404 			pqm->process->pasid, type, retval);
405 		goto err_create_queue;
406 	}
407 
408 	if (q && p_doorbell_offset_in_process) {
409 		/* Return the doorbell offset within the doorbell page
410 		 * to the caller so it can be passed up to user mode
411 		 * (in bytes).
412 		 * relative doorbell index = Absolute doorbell index -
413 		 * absolute index of first doorbell in the page.
414 		 */
415 		uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,
416 								       pdd->qpd.proc_doorbells,
417 								       0,
418 								       pdd->dev->kfd->device_info.doorbell_size);
419 
420 		*p_doorbell_offset_in_process = (q->properties.doorbell_off
421 						- first_db_index) * sizeof(uint32_t);
422 	}
423 
424 	pr_debug("PQM After DQM create queue\n");
425 
426 	list_add(&pqn->process_queue_list, &pqm->queues);
427 
428 	if (q) {
429 		pr_debug("PQM done creating queue\n");
430 		kfd_procfs_add_queue(q);
431 		print_queue_properties(&q->properties);
432 	}
433 
434 	return retval;
435 
436 err_create_queue:
437 	uninit_queue(q);
438 	if (kq)
439 		kernel_queue_uninit(kq, false);
440 	kfree(pqn);
441 err_allocate_pqn:
442 	/* check if queues list is empty unregister process from device */
443 	clear_bit(*qid, pqm->queue_slot_bitmap);
444 	if (list_empty(&pdd->qpd.queues_list) &&
445 	    list_empty(&pdd->qpd.priv_queue_list))
446 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
447 	return retval;
448 }
449 
pqm_destroy_queue(struct process_queue_manager * pqm,unsigned int qid)450 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
451 {
452 	struct process_queue_node *pqn;
453 	struct kfd_process_device *pdd;
454 	struct device_queue_manager *dqm;
455 	struct kfd_node *dev;
456 	int retval;
457 
458 	dqm = NULL;
459 
460 	retval = 0;
461 
462 	pqn = get_queue_by_qid(pqm, qid);
463 	if (!pqn) {
464 		pr_err("Queue id does not match any known queue\n");
465 		return -EINVAL;
466 	}
467 
468 	dev = NULL;
469 	if (pqn->kq)
470 		dev = pqn->kq->dev;
471 	if (pqn->q)
472 		dev = pqn->q->device;
473 	if (WARN_ON(!dev))
474 		return -ENODEV;
475 
476 	pdd = kfd_get_process_device_data(dev, pqm->process);
477 	if (!pdd) {
478 		pr_err("Process device data doesn't exist\n");
479 		return -1;
480 	}
481 
482 	if (pqn->kq) {
483 		/* destroy kernel queue (DIQ) */
484 		dqm = pqn->kq->dev->dqm;
485 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
486 		kernel_queue_uninit(pqn->kq, false);
487 	}
488 
489 	if (pqn->q) {
490 		kfd_procfs_del_queue(pqn->q);
491 		dqm = pqn->q->device->dqm;
492 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
493 		if (retval) {
494 			pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",
495 				pqm->process->pasid,
496 				pqn->q->properties.queue_id, retval);
497 			if (retval != -ETIME)
498 				goto err_destroy_queue;
499 		}
500 
501 		pqm_clean_queue_resource(pqm, pqn);
502 		uninit_queue(pqn->q);
503 	}
504 
505 	list_del(&pqn->process_queue_list);
506 	kfree(pqn);
507 	clear_bit(qid, pqm->queue_slot_bitmap);
508 
509 	if (list_empty(&pdd->qpd.queues_list) &&
510 	    list_empty(&pdd->qpd.priv_queue_list))
511 		dqm->ops.unregister_process(dqm, &pdd->qpd);
512 
513 err_destroy_queue:
514 	return retval;
515 }
516 
pqm_update_queue_properties(struct process_queue_manager * pqm,unsigned int qid,struct queue_properties * p)517 int pqm_update_queue_properties(struct process_queue_manager *pqm,
518 				unsigned int qid, struct queue_properties *p)
519 {
520 	int retval;
521 	struct process_queue_node *pqn;
522 
523 	pqn = get_queue_by_qid(pqm, qid);
524 	if (!pqn) {
525 		pr_debug("No queue %d exists for update operation\n", qid);
526 		return -EFAULT;
527 	}
528 
529 	pqn->q->properties.queue_address = p->queue_address;
530 	pqn->q->properties.queue_size = p->queue_size;
531 	pqn->q->properties.queue_percent = p->queue_percent;
532 	pqn->q->properties.priority = p->priority;
533 	pqn->q->properties.pm4_target_xcc = p->pm4_target_xcc;
534 
535 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
536 							pqn->q, NULL);
537 	if (retval != 0)
538 		return retval;
539 
540 	return 0;
541 }
542 
pqm_update_mqd(struct process_queue_manager * pqm,unsigned int qid,struct mqd_update_info * minfo)543 int pqm_update_mqd(struct process_queue_manager *pqm,
544 				unsigned int qid, struct mqd_update_info *minfo)
545 {
546 	int retval;
547 	struct process_queue_node *pqn;
548 
549 	pqn = get_queue_by_qid(pqm, qid);
550 	if (!pqn) {
551 		pr_debug("No queue %d exists for update operation\n", qid);
552 		return -EFAULT;
553 	}
554 
555 	/* CUs are masked for debugger requirements so deny user mask  */
556 	if (pqn->q->properties.is_dbg_wa && minfo && minfo->cu_mask.ptr)
557 		return -EBUSY;
558 
559 	/* ASICs that have WGPs must enforce pairwise enabled mask checks. */
560 	if (minfo && minfo->cu_mask.ptr &&
561 			KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0)) {
562 		int i;
563 
564 		for (i = 0; i < minfo->cu_mask.count; i += 2) {
565 			uint32_t cu_pair = (minfo->cu_mask.ptr[i / 32] >> (i % 32)) & 0x3;
566 
567 			if (cu_pair && cu_pair != 0x3) {
568 				pr_debug("CUs must be adjacent pairwise enabled.\n");
569 				return -EINVAL;
570 			}
571 		}
572 	}
573 
574 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
575 							pqn->q, minfo);
576 	if (retval != 0)
577 		return retval;
578 
579 	if (minfo && minfo->cu_mask.ptr)
580 		pqn->q->properties.is_user_cu_masked = true;
581 
582 	return 0;
583 }
584 
pqm_get_kernel_queue(struct process_queue_manager * pqm,unsigned int qid)585 struct kernel_queue *pqm_get_kernel_queue(
586 					struct process_queue_manager *pqm,
587 					unsigned int qid)
588 {
589 	struct process_queue_node *pqn;
590 
591 	pqn = get_queue_by_qid(pqm, qid);
592 	if (pqn && pqn->kq)
593 		return pqn->kq;
594 
595 	return NULL;
596 }
597 
pqm_get_user_queue(struct process_queue_manager * pqm,unsigned int qid)598 struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
599 					unsigned int qid)
600 {
601 	struct process_queue_node *pqn;
602 
603 	pqn = get_queue_by_qid(pqm, qid);
604 	return pqn ? pqn->q : NULL;
605 }
606 
pqm_get_wave_state(struct process_queue_manager * pqm,unsigned int qid,void __user * ctl_stack,u32 * ctl_stack_used_size,u32 * save_area_used_size)607 int pqm_get_wave_state(struct process_queue_manager *pqm,
608 		       unsigned int qid,
609 		       void __user *ctl_stack,
610 		       u32 *ctl_stack_used_size,
611 		       u32 *save_area_used_size)
612 {
613 	struct process_queue_node *pqn;
614 
615 	pqn = get_queue_by_qid(pqm, qid);
616 	if (!pqn) {
617 		pr_debug("amdkfd: No queue %d exists for operation\n",
618 			 qid);
619 		return -EFAULT;
620 	}
621 
622 	return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
623 						       pqn->q,
624 						       ctl_stack,
625 						       ctl_stack_used_size,
626 						       save_area_used_size);
627 }
628 
pqm_get_queue_snapshot(struct process_queue_manager * pqm,uint64_t exception_clear_mask,void __user * buf,int * num_qss_entries,uint32_t * entry_size)629 int pqm_get_queue_snapshot(struct process_queue_manager *pqm,
630 			   uint64_t exception_clear_mask,
631 			   void __user *buf,
632 			   int *num_qss_entries,
633 			   uint32_t *entry_size)
634 {
635 	struct process_queue_node *pqn;
636 	struct kfd_queue_snapshot_entry src;
637 	uint32_t tmp_entry_size = *entry_size, tmp_qss_entries = *num_qss_entries;
638 	int r = 0;
639 
640 	*num_qss_entries = 0;
641 	if (!(*entry_size))
642 		return -EINVAL;
643 
644 	*entry_size = min_t(size_t, *entry_size, sizeof(struct kfd_queue_snapshot_entry));
645 	mutex_lock(&pqm->process->event_mutex);
646 
647 	memset(&src, 0, sizeof(src));
648 
649 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
650 		if (!pqn->q)
651 			continue;
652 
653 		if (*num_qss_entries < tmp_qss_entries) {
654 			set_queue_snapshot_entry(pqn->q, exception_clear_mask, &src);
655 
656 			if (copy_to_user(buf, &src, *entry_size)) {
657 				r = -EFAULT;
658 				break;
659 			}
660 			buf += tmp_entry_size;
661 		}
662 		*num_qss_entries += 1;
663 	}
664 
665 	mutex_unlock(&pqm->process->event_mutex);
666 	return r;
667 }
668 
get_queue_data_sizes(struct kfd_process_device * pdd,struct queue * q,uint32_t * mqd_size,uint32_t * ctl_stack_size)669 static int get_queue_data_sizes(struct kfd_process_device *pdd,
670 				struct queue *q,
671 				uint32_t *mqd_size,
672 				uint32_t *ctl_stack_size)
673 {
674 	int ret;
675 
676 	ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,
677 					    q->properties.queue_id,
678 					    mqd_size,
679 					    ctl_stack_size);
680 	if (ret)
681 		pr_err("Failed to get queue dump info (%d)\n", ret);
682 
683 	return ret;
684 }
685 
kfd_process_get_queue_info(struct kfd_process * p,uint32_t * num_queues,uint64_t * priv_data_sizes)686 int kfd_process_get_queue_info(struct kfd_process *p,
687 			       uint32_t *num_queues,
688 			       uint64_t *priv_data_sizes)
689 {
690 	uint32_t extra_data_sizes = 0;
691 	struct queue *q;
692 	int i;
693 	int ret;
694 
695 	*num_queues = 0;
696 
697 	/* Run over all PDDs of the process */
698 	for (i = 0; i < p->n_pdds; i++) {
699 		struct kfd_process_device *pdd = p->pdds[i];
700 
701 		list_for_each_entry(q, &pdd->qpd.queues_list, list) {
702 			if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
703 				q->properties.type == KFD_QUEUE_TYPE_SDMA ||
704 				q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
705 				uint32_t mqd_size, ctl_stack_size;
706 
707 				*num_queues = *num_queues + 1;
708 
709 				ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
710 				if (ret)
711 					return ret;
712 
713 				extra_data_sizes += mqd_size + ctl_stack_size;
714 			} else {
715 				pr_err("Unsupported queue type (%d)\n", q->properties.type);
716 				return -EOPNOTSUPP;
717 			}
718 		}
719 	}
720 	*priv_data_sizes = extra_data_sizes +
721 				(*num_queues * sizeof(struct kfd_criu_queue_priv_data));
722 
723 	return 0;
724 }
725 
pqm_checkpoint_mqd(struct process_queue_manager * pqm,unsigned int qid,void * mqd,void * ctl_stack)726 static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,
727 			      unsigned int qid,
728 			      void *mqd,
729 			      void *ctl_stack)
730 {
731 	struct process_queue_node *pqn;
732 
733 	pqn = get_queue_by_qid(pqm, qid);
734 	if (!pqn) {
735 		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
736 		return -EFAULT;
737 	}
738 
739 	if (!pqn->q->device->dqm->ops.checkpoint_mqd) {
740 		pr_err("amdkfd: queue dumping not supported on this device\n");
741 		return -EOPNOTSUPP;
742 	}
743 
744 	return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,
745 						       pqn->q, mqd, ctl_stack);
746 }
747 
criu_checkpoint_queue(struct kfd_process_device * pdd,struct queue * q,struct kfd_criu_queue_priv_data * q_data)748 static int criu_checkpoint_queue(struct kfd_process_device *pdd,
749 			   struct queue *q,
750 			   struct kfd_criu_queue_priv_data *q_data)
751 {
752 	uint8_t *mqd, *ctl_stack;
753 	int ret;
754 
755 	mqd = (void *)(q_data + 1);
756 	ctl_stack = mqd + q_data->mqd_size;
757 
758 	q_data->gpu_id = pdd->user_gpu_id;
759 	q_data->type = q->properties.type;
760 	q_data->format = q->properties.format;
761 	q_data->q_id =  q->properties.queue_id;
762 	q_data->q_address = q->properties.queue_address;
763 	q_data->q_size = q->properties.queue_size;
764 	q_data->priority = q->properties.priority;
765 	q_data->q_percent = q->properties.queue_percent;
766 	q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr;
767 	q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr;
768 	q_data->doorbell_id = q->doorbell_id;
769 
770 	q_data->sdma_id = q->sdma_id;
771 
772 	q_data->eop_ring_buffer_address =
773 		q->properties.eop_ring_buffer_address;
774 
775 	q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size;
776 
777 	q_data->ctx_save_restore_area_address =
778 		q->properties.ctx_save_restore_area_address;
779 
780 	q_data->ctx_save_restore_area_size =
781 		q->properties.ctx_save_restore_area_size;
782 
783 	q_data->gws = !!q->gws;
784 
785 	ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
786 	if (ret) {
787 		pr_err("Failed checkpoint queue_mqd (%d)\n", ret);
788 		return ret;
789 	}
790 
791 	pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);
792 	return ret;
793 }
794 
criu_checkpoint_queues_device(struct kfd_process_device * pdd,uint8_t __user * user_priv,unsigned int * q_index,uint64_t * queues_priv_data_offset)795 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
796 				   uint8_t __user *user_priv,
797 				   unsigned int *q_index,
798 				   uint64_t *queues_priv_data_offset)
799 {
800 	unsigned int q_private_data_size = 0;
801 	uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */
802 	struct queue *q;
803 	int ret = 0;
804 
805 	list_for_each_entry(q, &pdd->qpd.queues_list, list) {
806 		struct kfd_criu_queue_priv_data *q_data;
807 		uint64_t q_data_size;
808 		uint32_t mqd_size;
809 		uint32_t ctl_stack_size;
810 
811 		if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
812 			q->properties.type != KFD_QUEUE_TYPE_SDMA &&
813 			q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {
814 
815 			pr_err("Unsupported queue type (%d)\n", q->properties.type);
816 			ret = -EOPNOTSUPP;
817 			break;
818 		}
819 
820 		ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
821 		if (ret)
822 			break;
823 
824 		q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;
825 
826 		/* Increase local buffer space if needed */
827 		if (q_private_data_size < q_data_size) {
828 			kfree(q_private_data);
829 
830 			q_private_data = kzalloc(q_data_size, GFP_KERNEL);
831 			if (!q_private_data) {
832 				ret = -ENOMEM;
833 				break;
834 			}
835 			q_private_data_size = q_data_size;
836 		}
837 
838 		q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
839 
840 		/* data stored in this order: priv_data, mqd, ctl_stack */
841 		q_data->mqd_size = mqd_size;
842 		q_data->ctl_stack_size = ctl_stack_size;
843 
844 		ret = criu_checkpoint_queue(pdd, q, q_data);
845 		if (ret)
846 			break;
847 
848 		q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;
849 
850 		ret = copy_to_user(user_priv + *queues_priv_data_offset,
851 				q_data, q_data_size);
852 		if (ret) {
853 			ret = -EFAULT;
854 			break;
855 		}
856 		*queues_priv_data_offset += q_data_size;
857 		*q_index = *q_index + 1;
858 	}
859 
860 	kfree(q_private_data);
861 
862 	return ret;
863 }
864 
kfd_criu_checkpoint_queues(struct kfd_process * p,uint8_t __user * user_priv_data,uint64_t * priv_data_offset)865 int kfd_criu_checkpoint_queues(struct kfd_process *p,
866 			 uint8_t __user *user_priv_data,
867 			 uint64_t *priv_data_offset)
868 {
869 	int ret = 0, pdd_index, q_index = 0;
870 
871 	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
872 		struct kfd_process_device *pdd = p->pdds[pdd_index];
873 
874 		/*
875 		 * criu_checkpoint_queues_device will copy data to user and update q_index and
876 		 * queues_priv_data_offset
877 		 */
878 		ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,
879 					      priv_data_offset);
880 
881 		if (ret)
882 			break;
883 	}
884 
885 	return ret;
886 }
887 
set_queue_properties_from_criu(struct queue_properties * qp,struct kfd_criu_queue_priv_data * q_data)888 static void set_queue_properties_from_criu(struct queue_properties *qp,
889 					  struct kfd_criu_queue_priv_data *q_data)
890 {
891 	qp->is_interop = false;
892 	qp->queue_percent = q_data->q_percent;
893 	qp->priority = q_data->priority;
894 	qp->queue_address = q_data->q_address;
895 	qp->queue_size = q_data->q_size;
896 	qp->read_ptr = (uint32_t *) q_data->read_ptr_addr;
897 	qp->write_ptr = (uint32_t *) q_data->write_ptr_addr;
898 	qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address;
899 	qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size;
900 	qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address;
901 	qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size;
902 	qp->ctl_stack_size = q_data->ctl_stack_size;
903 	qp->type = q_data->type;
904 	qp->format = q_data->format;
905 }
906 
kfd_criu_restore_queue(struct kfd_process * p,uint8_t __user * user_priv_ptr,uint64_t * priv_data_offset,uint64_t max_priv_data_size)907 int kfd_criu_restore_queue(struct kfd_process *p,
908 			   uint8_t __user *user_priv_ptr,
909 			   uint64_t *priv_data_offset,
910 			   uint64_t max_priv_data_size)
911 {
912 	uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;
913 	struct kfd_criu_queue_priv_data *q_data;
914 	struct kfd_process_device *pdd;
915 	uint64_t q_extra_data_size;
916 	struct queue_properties qp;
917 	unsigned int queue_id;
918 	int ret = 0;
919 
920 	if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)
921 		return -EINVAL;
922 
923 	q_data = kmalloc(sizeof(*q_data), GFP_KERNEL);
924 	if (!q_data)
925 		return -ENOMEM;
926 
927 	ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data));
928 	if (ret) {
929 		ret = -EFAULT;
930 		goto exit;
931 	}
932 
933 	*priv_data_offset += sizeof(*q_data);
934 	q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size;
935 
936 	if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {
937 		ret = -EINVAL;
938 		goto exit;
939 	}
940 
941 	q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);
942 	if (!q_extra_data) {
943 		ret = -ENOMEM;
944 		goto exit;
945 	}
946 
947 	ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);
948 	if (ret) {
949 		ret = -EFAULT;
950 		goto exit;
951 	}
952 
953 	*priv_data_offset += q_extra_data_size;
954 
955 	pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);
956 	if (!pdd) {
957 		pr_err("Failed to get pdd\n");
958 		ret = -EINVAL;
959 		goto exit;
960 	}
961 
962 	/* data stored in this order: mqd, ctl_stack */
963 	mqd = q_extra_data;
964 	ctl_stack = mqd + q_data->mqd_size;
965 
966 	memset(&qp, 0, sizeof(qp));
967 	set_queue_properties_from_criu(&qp, q_data);
968 
969 	print_queue_properties(&qp);
970 
971 	ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, NULL, q_data, mqd, ctl_stack,
972 				NULL);
973 	if (ret) {
974 		pr_err("Failed to create new queue err:%d\n", ret);
975 		goto exit;
976 	}
977 
978 	if (q_data->gws)
979 		ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);
980 
981 exit:
982 	if (ret)
983 		pr_err("Failed to restore queue (%d)\n", ret);
984 	else
985 		pr_debug("Queue id %d was restored successfully\n", queue_id);
986 
987 	kfree(q_data);
988 	kfree(q_extra_data);
989 
990 	return ret;
991 }
992 
pqm_get_queue_checkpoint_info(struct process_queue_manager * pqm,unsigned int qid,uint32_t * mqd_size,uint32_t * ctl_stack_size)993 int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
994 				  unsigned int qid,
995 				  uint32_t *mqd_size,
996 				  uint32_t *ctl_stack_size)
997 {
998 	struct process_queue_node *pqn;
999 
1000 	pqn = get_queue_by_qid(pqm, qid);
1001 	if (!pqn) {
1002 		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
1003 		return -EFAULT;
1004 	}
1005 
1006 	if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {
1007 		pr_err("amdkfd: queue dumping not supported on this device\n");
1008 		return -EOPNOTSUPP;
1009 	}
1010 
1011 	pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
1012 						       pqn->q, mqd_size,
1013 						       ctl_stack_size);
1014 	return 0;
1015 }
1016 
1017 #if defined(CONFIG_DEBUG_FS)
1018 
pqm_debugfs_mqds(struct seq_file * m,void * data)1019 int pqm_debugfs_mqds(struct seq_file *m, void *data)
1020 {
1021 	struct process_queue_manager *pqm = data;
1022 	struct process_queue_node *pqn;
1023 	struct queue *q;
1024 	enum KFD_MQD_TYPE mqd_type;
1025 	struct mqd_manager *mqd_mgr;
1026 	int r = 0, xcc, num_xccs = 1;
1027 	void *mqd;
1028 	uint64_t size = 0;
1029 
1030 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
1031 		if (pqn->q) {
1032 			q = pqn->q;
1033 			switch (q->properties.type) {
1034 			case KFD_QUEUE_TYPE_SDMA:
1035 			case KFD_QUEUE_TYPE_SDMA_XGMI:
1036 				seq_printf(m, "  SDMA queue on device %x\n",
1037 					   q->device->id);
1038 				mqd_type = KFD_MQD_TYPE_SDMA;
1039 				break;
1040 			case KFD_QUEUE_TYPE_COMPUTE:
1041 				seq_printf(m, "  Compute queue on device %x\n",
1042 					   q->device->id);
1043 				mqd_type = KFD_MQD_TYPE_CP;
1044 				num_xccs = NUM_XCC(q->device->xcc_mask);
1045 				break;
1046 			default:
1047 				seq_printf(m,
1048 				"  Bad user queue type %d on device %x\n",
1049 					   q->properties.type, q->device->id);
1050 				continue;
1051 			}
1052 			mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
1053 			size = mqd_mgr->mqd_stride(mqd_mgr,
1054 							&q->properties);
1055 		} else if (pqn->kq) {
1056 			q = pqn->kq->queue;
1057 			mqd_mgr = pqn->kq->mqd_mgr;
1058 			switch (q->properties.type) {
1059 			case KFD_QUEUE_TYPE_DIQ:
1060 				seq_printf(m, "  DIQ on device %x\n",
1061 					   pqn->kq->dev->id);
1062 				break;
1063 			default:
1064 				seq_printf(m,
1065 				"  Bad kernel queue type %d on device %x\n",
1066 					   q->properties.type,
1067 					   pqn->kq->dev->id);
1068 				continue;
1069 			}
1070 		} else {
1071 			seq_printf(m,
1072 		"  Weird: Queue node with neither kernel nor user queue\n");
1073 			continue;
1074 		}
1075 
1076 		for (xcc = 0; xcc < num_xccs; xcc++) {
1077 			mqd = q->mqd + size * xcc;
1078 			r = mqd_mgr->debugfs_show_mqd(m, mqd);
1079 			if (r != 0)
1080 				break;
1081 		}
1082 	}
1083 
1084 	return r;
1085 }
1086 
1087 #endif
1088