1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/slab.h>
26 #include <linux/list.h>
27 #include "kfd_device_queue_manager.h"
28 #include "kfd_priv.h"
29 #include "kfd_kernel_queue.h"
30 #include "amdgpu_amdkfd.h"
31 
32 static inline struct process_queue_node *get_queue_by_qid(
33 			struct process_queue_manager *pqm, unsigned int qid)
34 {
35 	struct process_queue_node *pqn;
36 
37 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
38 		if ((pqn->q && pqn->q->properties.queue_id == qid) ||
39 		    (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
40 			return pqn;
41 	}
42 
43 	return NULL;
44 }
45 
46 static int assign_queue_slot_by_qid(struct process_queue_manager *pqm,
47 				    unsigned int qid)
48 {
49 	if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
50 		return -EINVAL;
51 
52 	if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) {
53 		pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid);
54 		return -ENOSPC;
55 	}
56 
57 	return 0;
58 }
59 
60 static int find_available_queue_slot(struct process_queue_manager *pqm,
61 					unsigned int *qid)
62 {
63 	unsigned long found;
64 
65 	found = find_first_zero_bit(pqm->queue_slot_bitmap,
66 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
67 
68 	pr_debug("The new slot id %lu\n", found);
69 
70 	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
71 		pr_info("Cannot open more queues for process with pasid 0x%x\n",
72 				pqm->process->pasid);
73 		return -ENOMEM;
74 	}
75 
76 	set_bit(found, pqm->queue_slot_bitmap);
77 	*qid = found;
78 
79 	return 0;
80 }
81 
82 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
83 {
84 	struct kfd_dev *dev = pdd->dev;
85 
86 	if (pdd->already_dequeued)
87 		return;
88 
89 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
90 	pdd->already_dequeued = true;
91 }
92 
93 int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
94 			void *gws)
95 {
96 	struct kfd_dev *dev = NULL;
97 	struct process_queue_node *pqn;
98 	struct kfd_process_device *pdd;
99 	struct kgd_mem *mem = NULL;
100 	int ret;
101 
102 	pqn = get_queue_by_qid(pqm, qid);
103 	if (!pqn) {
104 		pr_err("Queue id does not match any known queue\n");
105 		return -EINVAL;
106 	}
107 
108 	if (pqn->q)
109 		dev = pqn->q->device;
110 	if (WARN_ON(!dev))
111 		return -ENODEV;
112 
113 	pdd = kfd_get_process_device_data(dev, pqm->process);
114 	if (!pdd) {
115 		pr_err("Process device data doesn't exist\n");
116 		return -EINVAL;
117 	}
118 
119 	/* Only allow one queue per process can have GWS assigned */
120 	if (gws && pdd->qpd.num_gws)
121 		return -EBUSY;
122 
123 	if (!gws && pdd->qpd.num_gws == 0)
124 		return -EINVAL;
125 
126 	if (gws)
127 		ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
128 			gws, &mem);
129 	else
130 		ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
131 			pqn->q->gws);
132 	if (unlikely(ret))
133 		return ret;
134 
135 	pqn->q->gws = mem;
136 	pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;
137 
138 	return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
139 							pqn->q, NULL);
140 }
141 
142 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
143 {
144 	int i;
145 
146 	for (i = 0; i < p->n_pdds; i++)
147 		kfd_process_dequeue_from_device(p->pdds[i]);
148 }
149 
150 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
151 {
152 	INIT_LIST_HEAD(&pqm->queues);
153 	pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
154 					       GFP_KERNEL);
155 	if (!pqm->queue_slot_bitmap)
156 		return -ENOMEM;
157 	pqm->process = p;
158 
159 	return 0;
160 }
161 
162 void pqm_uninit(struct process_queue_manager *pqm)
163 {
164 	struct process_queue_node *pqn, *next;
165 
166 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
167 		if (pqn->q && pqn->q->gws)
168 			amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
169 				pqn->q->gws);
170 		kfd_procfs_del_queue(pqn->q);
171 		uninit_queue(pqn->q);
172 		list_del(&pqn->process_queue_list);
173 		kfree(pqn);
174 	}
175 
176 	bitmap_free(pqm->queue_slot_bitmap);
177 	pqm->queue_slot_bitmap = NULL;
178 }
179 
180 static int init_user_queue(struct process_queue_manager *pqm,
181 				struct kfd_dev *dev, struct queue **q,
182 				struct queue_properties *q_properties,
183 				struct file *f, unsigned int qid)
184 {
185 	int retval;
186 
187 	/* Doorbell initialized in user space*/
188 	q_properties->doorbell_ptr = NULL;
189 
190 	/* let DQM handle it*/
191 	q_properties->vmid = 0;
192 	q_properties->queue_id = qid;
193 
194 	retval = init_queue(q, q_properties);
195 	if (retval != 0)
196 		return retval;
197 
198 	(*q)->device = dev;
199 	(*q)->process = pqm->process;
200 
201 	pr_debug("PQM After init queue");
202 
203 	return retval;
204 }
205 
206 int pqm_create_queue(struct process_queue_manager *pqm,
207 			    struct kfd_dev *dev,
208 			    struct file *f,
209 			    struct queue_properties *properties,
210 			    unsigned int *qid,
211 			    const struct kfd_criu_queue_priv_data *q_data,
212 			    const void *restore_mqd,
213 			    const void *restore_ctl_stack,
214 			    uint32_t *p_doorbell_offset_in_process)
215 {
216 	int retval;
217 	struct kfd_process_device *pdd;
218 	struct queue *q;
219 	struct process_queue_node *pqn;
220 	struct kernel_queue *kq;
221 	enum kfd_queue_type type = properties->type;
222 	unsigned int max_queues = 127; /* HWS limit */
223 
224 	q = NULL;
225 	kq = NULL;
226 
227 	pdd = kfd_get_process_device_data(dev, pqm->process);
228 	if (!pdd) {
229 		pr_err("Process device data doesn't exist\n");
230 		return -1;
231 	}
232 
233 	/*
234 	 * for debug process, verify that it is within the static queues limit
235 	 * currently limit is set to half of the total avail HQD slots
236 	 * If we are just about to create DIQ, the is_debug flag is not set yet
237 	 * Hence we also check the type as well
238 	 */
239 	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
240 		max_queues = dev->device_info.max_no_of_hqd/2;
241 
242 	if (pdd->qpd.queue_count >= max_queues)
243 		return -ENOSPC;
244 
245 	if (q_data) {
246 		retval = assign_queue_slot_by_qid(pqm, q_data->q_id);
247 		*qid = q_data->q_id;
248 	} else
249 		retval = find_available_queue_slot(pqm, qid);
250 
251 	if (retval != 0)
252 		return retval;
253 
254 	if (list_empty(&pdd->qpd.queues_list) &&
255 	    list_empty(&pdd->qpd.priv_queue_list))
256 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
257 
258 	pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
259 	if (!pqn) {
260 		retval = -ENOMEM;
261 		goto err_allocate_pqn;
262 	}
263 
264 	switch (type) {
265 	case KFD_QUEUE_TYPE_SDMA:
266 	case KFD_QUEUE_TYPE_SDMA_XGMI:
267 		/* SDMA queues are always allocated statically no matter
268 		 * which scheduler mode is used. We also do not need to
269 		 * check whether a SDMA queue can be allocated here, because
270 		 * allocate_sdma_queue() in create_queue() has the
271 		 * corresponding check logic.
272 		 */
273 		retval = init_user_queue(pqm, dev, &q, properties, f, *qid);
274 		if (retval != 0)
275 			goto err_create_queue;
276 		pqn->q = q;
277 		pqn->kq = NULL;
278 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
279 						    restore_mqd, restore_ctl_stack);
280 		print_queue(q);
281 		break;
282 
283 	case KFD_QUEUE_TYPE_COMPUTE:
284 		/* check if there is over subscription */
285 		if ((dev->dqm->sched_policy ==
286 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
287 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
288 		(dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {
289 			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
290 			retval = -EPERM;
291 			goto err_create_queue;
292 		}
293 
294 		retval = init_user_queue(pqm, dev, &q, properties, f, *qid);
295 		if (retval != 0)
296 			goto err_create_queue;
297 		pqn->q = q;
298 		pqn->kq = NULL;
299 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
300 						    restore_mqd, restore_ctl_stack);
301 		print_queue(q);
302 		break;
303 	case KFD_QUEUE_TYPE_DIQ:
304 		kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
305 		if (!kq) {
306 			retval = -ENOMEM;
307 			goto err_create_queue;
308 		}
309 		kq->queue->properties.queue_id = *qid;
310 		pqn->kq = kq;
311 		pqn->q = NULL;
312 		retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
313 							kq, &pdd->qpd);
314 		break;
315 	default:
316 		WARN(1, "Invalid queue type %d", type);
317 		retval = -EINVAL;
318 	}
319 
320 	if (retval != 0) {
321 		pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n",
322 			pqm->process->pasid, type, retval);
323 		goto err_create_queue;
324 	}
325 
326 	if (q && p_doorbell_offset_in_process)
327 		/* Return the doorbell offset within the doorbell page
328 		 * to the caller so it can be passed up to user mode
329 		 * (in bytes).
330 		 * There are always 1024 doorbells per process, so in case
331 		 * of 8-byte doorbells, there are two doorbell pages per
332 		 * process.
333 		 */
334 		*p_doorbell_offset_in_process =
335 			(q->properties.doorbell_off * sizeof(uint32_t)) &
336 			(kfd_doorbell_process_slice(dev) - 1);
337 
338 	pr_debug("PQM After DQM create queue\n");
339 
340 	list_add(&pqn->process_queue_list, &pqm->queues);
341 
342 	if (q) {
343 		pr_debug("PQM done creating queue\n");
344 		kfd_procfs_add_queue(q);
345 		print_queue_properties(&q->properties);
346 	}
347 
348 	return retval;
349 
350 err_create_queue:
351 	uninit_queue(q);
352 	if (kq)
353 		kernel_queue_uninit(kq, false);
354 	kfree(pqn);
355 err_allocate_pqn:
356 	/* check if queues list is empty unregister process from device */
357 	clear_bit(*qid, pqm->queue_slot_bitmap);
358 	if (list_empty(&pdd->qpd.queues_list) &&
359 	    list_empty(&pdd->qpd.priv_queue_list))
360 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
361 	return retval;
362 }
363 
364 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
365 {
366 	struct process_queue_node *pqn;
367 	struct kfd_process_device *pdd;
368 	struct device_queue_manager *dqm;
369 	struct kfd_dev *dev;
370 	int retval;
371 
372 	dqm = NULL;
373 
374 	retval = 0;
375 
376 	pqn = get_queue_by_qid(pqm, qid);
377 	if (!pqn) {
378 		pr_err("Queue id does not match any known queue\n");
379 		return -EINVAL;
380 	}
381 
382 	dev = NULL;
383 	if (pqn->kq)
384 		dev = pqn->kq->dev;
385 	if (pqn->q)
386 		dev = pqn->q->device;
387 	if (WARN_ON(!dev))
388 		return -ENODEV;
389 
390 	pdd = kfd_get_process_device_data(dev, pqm->process);
391 	if (!pdd) {
392 		pr_err("Process device data doesn't exist\n");
393 		return -1;
394 	}
395 
396 	if (pqn->kq) {
397 		/* destroy kernel queue (DIQ) */
398 		dqm = pqn->kq->dev->dqm;
399 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
400 		kernel_queue_uninit(pqn->kq, false);
401 	}
402 
403 	if (pqn->q) {
404 		kfd_procfs_del_queue(pqn->q);
405 		dqm = pqn->q->device->dqm;
406 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
407 		if (retval) {
408 			pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",
409 				pqm->process->pasid,
410 				pqn->q->properties.queue_id, retval);
411 			if (retval != -ETIME)
412 				goto err_destroy_queue;
413 		}
414 
415 		if (pqn->q->gws) {
416 			amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
417 				pqn->q->gws);
418 			pdd->qpd.num_gws = 0;
419 		}
420 
421 		uninit_queue(pqn->q);
422 	}
423 
424 	list_del(&pqn->process_queue_list);
425 	kfree(pqn);
426 	clear_bit(qid, pqm->queue_slot_bitmap);
427 
428 	if (list_empty(&pdd->qpd.queues_list) &&
429 	    list_empty(&pdd->qpd.priv_queue_list))
430 		dqm->ops.unregister_process(dqm, &pdd->qpd);
431 
432 err_destroy_queue:
433 	return retval;
434 }
435 
436 int pqm_update_queue_properties(struct process_queue_manager *pqm,
437 				unsigned int qid, struct queue_properties *p)
438 {
439 	int retval;
440 	struct process_queue_node *pqn;
441 
442 	pqn = get_queue_by_qid(pqm, qid);
443 	if (!pqn) {
444 		pr_debug("No queue %d exists for update operation\n", qid);
445 		return -EFAULT;
446 	}
447 
448 	pqn->q->properties.queue_address = p->queue_address;
449 	pqn->q->properties.queue_size = p->queue_size;
450 	pqn->q->properties.queue_percent = p->queue_percent;
451 	pqn->q->properties.priority = p->priority;
452 
453 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
454 							pqn->q, NULL);
455 	if (retval != 0)
456 		return retval;
457 
458 	return 0;
459 }
460 
461 int pqm_update_mqd(struct process_queue_manager *pqm,
462 				unsigned int qid, struct mqd_update_info *minfo)
463 {
464 	int retval;
465 	struct process_queue_node *pqn;
466 
467 	pqn = get_queue_by_qid(pqm, qid);
468 	if (!pqn) {
469 		pr_debug("No queue %d exists for update operation\n", qid);
470 		return -EFAULT;
471 	}
472 
473 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
474 							pqn->q, minfo);
475 	if (retval != 0)
476 		return retval;
477 
478 	return 0;
479 }
480 
481 struct kernel_queue *pqm_get_kernel_queue(
482 					struct process_queue_manager *pqm,
483 					unsigned int qid)
484 {
485 	struct process_queue_node *pqn;
486 
487 	pqn = get_queue_by_qid(pqm, qid);
488 	if (pqn && pqn->kq)
489 		return pqn->kq;
490 
491 	return NULL;
492 }
493 
494 struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
495 					unsigned int qid)
496 {
497 	struct process_queue_node *pqn;
498 
499 	pqn = get_queue_by_qid(pqm, qid);
500 	return pqn ? pqn->q : NULL;
501 }
502 
503 int pqm_get_wave_state(struct process_queue_manager *pqm,
504 		       unsigned int qid,
505 		       void __user *ctl_stack,
506 		       u32 *ctl_stack_used_size,
507 		       u32 *save_area_used_size)
508 {
509 	struct process_queue_node *pqn;
510 
511 	pqn = get_queue_by_qid(pqm, qid);
512 	if (!pqn) {
513 		pr_debug("amdkfd: No queue %d exists for operation\n",
514 			 qid);
515 		return -EFAULT;
516 	}
517 
518 	return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
519 						       pqn->q,
520 						       ctl_stack,
521 						       ctl_stack_used_size,
522 						       save_area_used_size);
523 }
524 
525 static int get_queue_data_sizes(struct kfd_process_device *pdd,
526 				struct queue *q,
527 				uint32_t *mqd_size,
528 				uint32_t *ctl_stack_size)
529 {
530 	int ret;
531 
532 	ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,
533 					    q->properties.queue_id,
534 					    mqd_size,
535 					    ctl_stack_size);
536 	if (ret)
537 		pr_err("Failed to get queue dump info (%d)\n", ret);
538 
539 	return ret;
540 }
541 
542 int kfd_process_get_queue_info(struct kfd_process *p,
543 			       uint32_t *num_queues,
544 			       uint64_t *priv_data_sizes)
545 {
546 	uint32_t extra_data_sizes = 0;
547 	struct queue *q;
548 	int i;
549 	int ret;
550 
551 	*num_queues = 0;
552 
553 	/* Run over all PDDs of the process */
554 	for (i = 0; i < p->n_pdds; i++) {
555 		struct kfd_process_device *pdd = p->pdds[i];
556 
557 		list_for_each_entry(q, &pdd->qpd.queues_list, list) {
558 			if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
559 				q->properties.type == KFD_QUEUE_TYPE_SDMA ||
560 				q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
561 				uint32_t mqd_size, ctl_stack_size;
562 
563 				*num_queues = *num_queues + 1;
564 
565 				ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
566 				if (ret)
567 					return ret;
568 
569 				extra_data_sizes += mqd_size + ctl_stack_size;
570 			} else {
571 				pr_err("Unsupported queue type (%d)\n", q->properties.type);
572 				return -EOPNOTSUPP;
573 			}
574 		}
575 	}
576 	*priv_data_sizes = extra_data_sizes +
577 				(*num_queues * sizeof(struct kfd_criu_queue_priv_data));
578 
579 	return 0;
580 }
581 
582 static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,
583 			      unsigned int qid,
584 			      void *mqd,
585 			      void *ctl_stack)
586 {
587 	struct process_queue_node *pqn;
588 
589 	pqn = get_queue_by_qid(pqm, qid);
590 	if (!pqn) {
591 		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
592 		return -EFAULT;
593 	}
594 
595 	if (!pqn->q->device->dqm->ops.checkpoint_mqd) {
596 		pr_err("amdkfd: queue dumping not supported on this device\n");
597 		return -EOPNOTSUPP;
598 	}
599 
600 	return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,
601 						       pqn->q, mqd, ctl_stack);
602 }
603 
604 static int criu_checkpoint_queue(struct kfd_process_device *pdd,
605 			   struct queue *q,
606 			   struct kfd_criu_queue_priv_data *q_data)
607 {
608 	uint8_t *mqd, *ctl_stack;
609 	int ret;
610 
611 	mqd = (void *)(q_data + 1);
612 	ctl_stack = mqd + q_data->mqd_size;
613 
614 	q_data->gpu_id = pdd->user_gpu_id;
615 	q_data->type = q->properties.type;
616 	q_data->format = q->properties.format;
617 	q_data->q_id =  q->properties.queue_id;
618 	q_data->q_address = q->properties.queue_address;
619 	q_data->q_size = q->properties.queue_size;
620 	q_data->priority = q->properties.priority;
621 	q_data->q_percent = q->properties.queue_percent;
622 	q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr;
623 	q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr;
624 	q_data->doorbell_id = q->doorbell_id;
625 
626 	q_data->sdma_id = q->sdma_id;
627 
628 	q_data->eop_ring_buffer_address =
629 		q->properties.eop_ring_buffer_address;
630 
631 	q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size;
632 
633 	q_data->ctx_save_restore_area_address =
634 		q->properties.ctx_save_restore_area_address;
635 
636 	q_data->ctx_save_restore_area_size =
637 		q->properties.ctx_save_restore_area_size;
638 
639 	ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
640 	if (ret) {
641 		pr_err("Failed checkpoint queue_mqd (%d)\n", ret);
642 		return ret;
643 	}
644 
645 	pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);
646 	return ret;
647 }
648 
649 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
650 				   uint8_t __user *user_priv,
651 				   unsigned int *q_index,
652 				   uint64_t *queues_priv_data_offset)
653 {
654 	unsigned int q_private_data_size = 0;
655 	uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */
656 	struct queue *q;
657 	int ret = 0;
658 
659 	list_for_each_entry(q, &pdd->qpd.queues_list, list) {
660 		struct kfd_criu_queue_priv_data *q_data;
661 		uint64_t q_data_size;
662 		uint32_t mqd_size;
663 		uint32_t ctl_stack_size;
664 
665 		if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
666 			q->properties.type != KFD_QUEUE_TYPE_SDMA &&
667 			q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {
668 
669 			pr_err("Unsupported queue type (%d)\n", q->properties.type);
670 			ret = -EOPNOTSUPP;
671 			break;
672 		}
673 
674 		ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
675 		if (ret)
676 			break;
677 
678 		q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;
679 
680 		/* Increase local buffer space if needed */
681 		if (q_private_data_size < q_data_size) {
682 			kfree(q_private_data);
683 
684 			q_private_data = kzalloc(q_data_size, GFP_KERNEL);
685 			if (!q_private_data) {
686 				ret = -ENOMEM;
687 				break;
688 			}
689 			q_private_data_size = q_data_size;
690 		}
691 
692 		q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
693 
694 		/* data stored in this order: priv_data, mqd, ctl_stack */
695 		q_data->mqd_size = mqd_size;
696 		q_data->ctl_stack_size = ctl_stack_size;
697 
698 		ret = criu_checkpoint_queue(pdd, q, q_data);
699 		if (ret)
700 			break;
701 
702 		q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;
703 
704 		ret = copy_to_user(user_priv + *queues_priv_data_offset,
705 				q_data, q_data_size);
706 		if (ret) {
707 			ret = -EFAULT;
708 			break;
709 		}
710 		*queues_priv_data_offset += q_data_size;
711 		*q_index = *q_index + 1;
712 	}
713 
714 	kfree(q_private_data);
715 
716 	return ret;
717 }
718 
719 int kfd_criu_checkpoint_queues(struct kfd_process *p,
720 			 uint8_t __user *user_priv_data,
721 			 uint64_t *priv_data_offset)
722 {
723 	int ret = 0, pdd_index, q_index = 0;
724 
725 	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
726 		struct kfd_process_device *pdd = p->pdds[pdd_index];
727 
728 		/*
729 		 * criu_checkpoint_queues_device will copy data to user and update q_index and
730 		 * queues_priv_data_offset
731 		 */
732 		ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,
733 					      priv_data_offset);
734 
735 		if (ret)
736 			break;
737 	}
738 
739 	return ret;
740 }
741 
742 static void set_queue_properties_from_criu(struct queue_properties *qp,
743 					  struct kfd_criu_queue_priv_data *q_data)
744 {
745 	qp->is_interop = false;
746 	qp->is_gws = q_data->is_gws;
747 	qp->queue_percent = q_data->q_percent;
748 	qp->priority = q_data->priority;
749 	qp->queue_address = q_data->q_address;
750 	qp->queue_size = q_data->q_size;
751 	qp->read_ptr = (uint32_t *) q_data->read_ptr_addr;
752 	qp->write_ptr = (uint32_t *) q_data->write_ptr_addr;
753 	qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address;
754 	qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size;
755 	qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address;
756 	qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size;
757 	qp->ctl_stack_size = q_data->ctl_stack_size;
758 	qp->type = q_data->type;
759 	qp->format = q_data->format;
760 }
761 
762 int kfd_criu_restore_queue(struct kfd_process *p,
763 			   uint8_t __user *user_priv_ptr,
764 			   uint64_t *priv_data_offset,
765 			   uint64_t max_priv_data_size)
766 {
767 	uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;
768 	struct kfd_criu_queue_priv_data *q_data;
769 	struct kfd_process_device *pdd;
770 	uint64_t q_extra_data_size;
771 	struct queue_properties qp;
772 	unsigned int queue_id;
773 	int ret = 0;
774 
775 	if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)
776 		return -EINVAL;
777 
778 	q_data = kmalloc(sizeof(*q_data), GFP_KERNEL);
779 	if (!q_data)
780 		return -ENOMEM;
781 
782 	ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data));
783 	if (ret) {
784 		ret = -EFAULT;
785 		goto exit;
786 	}
787 
788 	*priv_data_offset += sizeof(*q_data);
789 	q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size;
790 
791 	if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {
792 		ret = -EINVAL;
793 		goto exit;
794 	}
795 
796 	q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);
797 	if (!q_extra_data) {
798 		ret = -ENOMEM;
799 		goto exit;
800 	}
801 
802 	ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);
803 	if (ret) {
804 		ret = -EFAULT;
805 		goto exit;
806 	}
807 
808 	*priv_data_offset += q_extra_data_size;
809 
810 	pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);
811 	if (!pdd) {
812 		pr_err("Failed to get pdd\n");
813 		ret = -EINVAL;
814 		goto exit;
815 	}
816 	/* data stored in this order: mqd, ctl_stack */
817 	mqd = q_extra_data;
818 	ctl_stack = mqd + q_data->mqd_size;
819 
820 	memset(&qp, 0, sizeof(qp));
821 	set_queue_properties_from_criu(&qp, q_data);
822 
823 	print_queue_properties(&qp);
824 
825 	ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, ctl_stack,
826 				NULL);
827 	if (ret) {
828 		pr_err("Failed to create new queue err:%d\n", ret);
829 		ret = -EINVAL;
830 	}
831 
832 exit:
833 	if (ret)
834 		pr_err("Failed to create queue (%d)\n", ret);
835 	else
836 		pr_debug("Queue id %d was restored successfully\n", queue_id);
837 
838 	kfree(q_data);
839 
840 	return ret;
841 }
842 
843 int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
844 				  unsigned int qid,
845 				  uint32_t *mqd_size,
846 				  uint32_t *ctl_stack_size)
847 {
848 	struct process_queue_node *pqn;
849 
850 	pqn = get_queue_by_qid(pqm, qid);
851 	if (!pqn) {
852 		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
853 		return -EFAULT;
854 	}
855 
856 	if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {
857 		pr_err("amdkfd: queue dumping not supported on this device\n");
858 		return -EOPNOTSUPP;
859 	}
860 
861 	pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
862 						       pqn->q, mqd_size,
863 						       ctl_stack_size);
864 	return 0;
865 }
866 
867 #if defined(CONFIG_DEBUG_FS)
868 
869 int pqm_debugfs_mqds(struct seq_file *m, void *data)
870 {
871 	struct process_queue_manager *pqm = data;
872 	struct process_queue_node *pqn;
873 	struct queue *q;
874 	enum KFD_MQD_TYPE mqd_type;
875 	struct mqd_manager *mqd_mgr;
876 	int r = 0;
877 
878 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
879 		if (pqn->q) {
880 			q = pqn->q;
881 			switch (q->properties.type) {
882 			case KFD_QUEUE_TYPE_SDMA:
883 			case KFD_QUEUE_TYPE_SDMA_XGMI:
884 				seq_printf(m, "  SDMA queue on device %x\n",
885 					   q->device->id);
886 				mqd_type = KFD_MQD_TYPE_SDMA;
887 				break;
888 			case KFD_QUEUE_TYPE_COMPUTE:
889 				seq_printf(m, "  Compute queue on device %x\n",
890 					   q->device->id);
891 				mqd_type = KFD_MQD_TYPE_CP;
892 				break;
893 			default:
894 				seq_printf(m,
895 				"  Bad user queue type %d on device %x\n",
896 					   q->properties.type, q->device->id);
897 				continue;
898 			}
899 			mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
900 		} else if (pqn->kq) {
901 			q = pqn->kq->queue;
902 			mqd_mgr = pqn->kq->mqd_mgr;
903 			switch (q->properties.type) {
904 			case KFD_QUEUE_TYPE_DIQ:
905 				seq_printf(m, "  DIQ on device %x\n",
906 					   pqn->kq->dev->id);
907 				break;
908 			default:
909 				seq_printf(m,
910 				"  Bad kernel queue type %d on device %x\n",
911 					   q->properties.type,
912 					   pqn->kq->dev->id);
913 				continue;
914 			}
915 		} else {
916 			seq_printf(m,
917 		"  Weird: Queue node with neither kernel nor user queue\n");
918 			continue;
919 		}
920 
921 		r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
922 		if (r != 0)
923 			break;
924 	}
925 
926 	return r;
927 }
928 
929 #endif
930