1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/slab.h> 26 #include <linux/list.h> 27 #include "kfd_device_queue_manager.h" 28 #include "kfd_priv.h" 29 #include "kfd_kernel_queue.h" 30 #include "amdgpu_amdkfd.h" 31 32 static inline struct process_queue_node *get_queue_by_qid( 33 struct process_queue_manager *pqm, unsigned int qid) 34 { 35 struct process_queue_node *pqn; 36 37 list_for_each_entry(pqn, &pqm->queues, process_queue_list) { 38 if ((pqn->q && pqn->q->properties.queue_id == qid) || 39 (pqn->kq && pqn->kq->queue->properties.queue_id == qid)) 40 return pqn; 41 } 42 43 return NULL; 44 } 45 46 static int assign_queue_slot_by_qid(struct process_queue_manager *pqm, 47 unsigned int qid) 48 { 49 if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 50 return -EINVAL; 51 52 if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) { 53 pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid); 54 return -ENOSPC; 55 } 56 57 return 0; 58 } 59 60 static int find_available_queue_slot(struct process_queue_manager *pqm, 61 unsigned int *qid) 62 { 63 unsigned long found; 64 65 found = find_first_zero_bit(pqm->queue_slot_bitmap, 66 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 67 68 pr_debug("The new slot id %lu\n", found); 69 70 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 71 pr_info("Cannot open more queues for process with pasid 0x%x\n", 72 pqm->process->pasid); 73 return -ENOMEM; 74 } 75 76 set_bit(found, pqm->queue_slot_bitmap); 77 *qid = found; 78 79 return 0; 80 } 81 82 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd) 83 { 84 struct kfd_dev *dev = pdd->dev; 85 86 if (pdd->already_dequeued) 87 return; 88 89 dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd); 90 pdd->already_dequeued = true; 91 } 92 93 int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid, 94 void *gws) 95 { 96 struct kfd_dev *dev = NULL; 97 struct process_queue_node *pqn; 98 struct kfd_process_device *pdd; 99 struct kgd_mem *mem = NULL; 100 int ret; 101 102 pqn = get_queue_by_qid(pqm, qid); 103 if (!pqn) { 104 pr_err("Queue id does not match any known queue\n"); 105 return -EINVAL; 106 } 107 108 if (pqn->q) 109 dev = pqn->q->device; 110 if (WARN_ON(!dev)) 111 return -ENODEV; 112 113 pdd = kfd_get_process_device_data(dev, pqm->process); 114 if (!pdd) { 115 pr_err("Process device data doesn't exist\n"); 116 return -EINVAL; 117 } 118 119 /* Only allow one queue per process can have GWS assigned */ 120 if (gws && pdd->qpd.num_gws) 121 return -EBUSY; 122 123 if (!gws && pdd->qpd.num_gws == 0) 124 return -EINVAL; 125 126 if (gws) 127 ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info, 128 gws, &mem); 129 else 130 ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info, 131 pqn->q->gws); 132 if (unlikely(ret)) 133 return ret; 134 135 pqn->q->gws = mem; 136 pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0; 137 138 return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, 139 pqn->q, NULL); 140 } 141 142 void kfd_process_dequeue_from_all_devices(struct kfd_process *p) 143 { 144 int i; 145 146 for (i = 0; i < p->n_pdds; i++) 147 kfd_process_dequeue_from_device(p->pdds[i]); 148 } 149 150 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p) 151 { 152 INIT_LIST_HEAD(&pqm->queues); 153 pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, 154 GFP_KERNEL); 155 if (!pqm->queue_slot_bitmap) 156 return -ENOMEM; 157 pqm->process = p; 158 159 return 0; 160 } 161 162 void pqm_uninit(struct process_queue_manager *pqm) 163 { 164 struct process_queue_node *pqn, *next; 165 166 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) { 167 if (pqn->q && pqn->q->gws) 168 amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info, 169 pqn->q->gws); 170 kfd_procfs_del_queue(pqn->q); 171 uninit_queue(pqn->q); 172 list_del(&pqn->process_queue_list); 173 kfree(pqn); 174 } 175 176 bitmap_free(pqm->queue_slot_bitmap); 177 pqm->queue_slot_bitmap = NULL; 178 } 179 180 static int init_user_queue(struct process_queue_manager *pqm, 181 struct kfd_dev *dev, struct queue **q, 182 struct queue_properties *q_properties, 183 struct file *f, unsigned int qid) 184 { 185 int retval; 186 187 /* Doorbell initialized in user space*/ 188 q_properties->doorbell_ptr = NULL; 189 190 /* let DQM handle it*/ 191 q_properties->vmid = 0; 192 q_properties->queue_id = qid; 193 194 retval = init_queue(q, q_properties); 195 if (retval != 0) 196 return retval; 197 198 (*q)->device = dev; 199 (*q)->process = pqm->process; 200 201 pr_debug("PQM After init queue"); 202 203 return retval; 204 } 205 206 int pqm_create_queue(struct process_queue_manager *pqm, 207 struct kfd_dev *dev, 208 struct file *f, 209 struct queue_properties *properties, 210 unsigned int *qid, 211 const struct kfd_criu_queue_priv_data *q_data, 212 const void *restore_mqd, 213 const void *restore_ctl_stack, 214 uint32_t *p_doorbell_offset_in_process) 215 { 216 int retval; 217 struct kfd_process_device *pdd; 218 struct queue *q; 219 struct process_queue_node *pqn; 220 struct kernel_queue *kq; 221 enum kfd_queue_type type = properties->type; 222 unsigned int max_queues = 127; /* HWS limit */ 223 224 q = NULL; 225 kq = NULL; 226 227 pdd = kfd_get_process_device_data(dev, pqm->process); 228 if (!pdd) { 229 pr_err("Process device data doesn't exist\n"); 230 return -1; 231 } 232 233 /* 234 * for debug process, verify that it is within the static queues limit 235 * currently limit is set to half of the total avail HQD slots 236 * If we are just about to create DIQ, the is_debug flag is not set yet 237 * Hence we also check the type as well 238 */ 239 if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ)) 240 max_queues = dev->device_info.max_no_of_hqd/2; 241 242 if (pdd->qpd.queue_count >= max_queues) 243 return -ENOSPC; 244 245 if (q_data) { 246 retval = assign_queue_slot_by_qid(pqm, q_data->q_id); 247 *qid = q_data->q_id; 248 } else 249 retval = find_available_queue_slot(pqm, qid); 250 251 if (retval != 0) 252 return retval; 253 254 if (list_empty(&pdd->qpd.queues_list) && 255 list_empty(&pdd->qpd.priv_queue_list)) 256 dev->dqm->ops.register_process(dev->dqm, &pdd->qpd); 257 258 pqn = kzalloc(sizeof(*pqn), GFP_KERNEL); 259 if (!pqn) { 260 retval = -ENOMEM; 261 goto err_allocate_pqn; 262 } 263 264 switch (type) { 265 case KFD_QUEUE_TYPE_SDMA: 266 case KFD_QUEUE_TYPE_SDMA_XGMI: 267 /* SDMA queues are always allocated statically no matter 268 * which scheduler mode is used. We also do not need to 269 * check whether a SDMA queue can be allocated here, because 270 * allocate_sdma_queue() in create_queue() has the 271 * corresponding check logic. 272 */ 273 retval = init_user_queue(pqm, dev, &q, properties, f, *qid); 274 if (retval != 0) 275 goto err_create_queue; 276 pqn->q = q; 277 pqn->kq = NULL; 278 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, 279 restore_mqd, restore_ctl_stack); 280 print_queue(q); 281 break; 282 283 case KFD_QUEUE_TYPE_COMPUTE: 284 /* check if there is over subscription */ 285 if ((dev->dqm->sched_policy == 286 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) && 287 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) || 288 (dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) { 289 pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n"); 290 retval = -EPERM; 291 goto err_create_queue; 292 } 293 294 retval = init_user_queue(pqm, dev, &q, properties, f, *qid); 295 if (retval != 0) 296 goto err_create_queue; 297 pqn->q = q; 298 pqn->kq = NULL; 299 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, 300 restore_mqd, restore_ctl_stack); 301 print_queue(q); 302 break; 303 case KFD_QUEUE_TYPE_DIQ: 304 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ); 305 if (!kq) { 306 retval = -ENOMEM; 307 goto err_create_queue; 308 } 309 kq->queue->properties.queue_id = *qid; 310 pqn->kq = kq; 311 pqn->q = NULL; 312 retval = dev->dqm->ops.create_kernel_queue(dev->dqm, 313 kq, &pdd->qpd); 314 break; 315 default: 316 WARN(1, "Invalid queue type %d", type); 317 retval = -EINVAL; 318 } 319 320 if (retval != 0) { 321 pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n", 322 pqm->process->pasid, type, retval); 323 goto err_create_queue; 324 } 325 326 if (q && p_doorbell_offset_in_process) 327 /* Return the doorbell offset within the doorbell page 328 * to the caller so it can be passed up to user mode 329 * (in bytes). 330 * There are always 1024 doorbells per process, so in case 331 * of 8-byte doorbells, there are two doorbell pages per 332 * process. 333 */ 334 *p_doorbell_offset_in_process = 335 (q->properties.doorbell_off * sizeof(uint32_t)) & 336 (kfd_doorbell_process_slice(dev) - 1); 337 338 pr_debug("PQM After DQM create queue\n"); 339 340 list_add(&pqn->process_queue_list, &pqm->queues); 341 342 if (q) { 343 pr_debug("PQM done creating queue\n"); 344 kfd_procfs_add_queue(q); 345 print_queue_properties(&q->properties); 346 } 347 348 return retval; 349 350 err_create_queue: 351 uninit_queue(q); 352 if (kq) 353 kernel_queue_uninit(kq, false); 354 kfree(pqn); 355 err_allocate_pqn: 356 /* check if queues list is empty unregister process from device */ 357 clear_bit(*qid, pqm->queue_slot_bitmap); 358 if (list_empty(&pdd->qpd.queues_list) && 359 list_empty(&pdd->qpd.priv_queue_list)) 360 dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd); 361 return retval; 362 } 363 364 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) 365 { 366 struct process_queue_node *pqn; 367 struct kfd_process_device *pdd; 368 struct device_queue_manager *dqm; 369 struct kfd_dev *dev; 370 int retval; 371 372 dqm = NULL; 373 374 retval = 0; 375 376 pqn = get_queue_by_qid(pqm, qid); 377 if (!pqn) { 378 pr_err("Queue id does not match any known queue\n"); 379 return -EINVAL; 380 } 381 382 dev = NULL; 383 if (pqn->kq) 384 dev = pqn->kq->dev; 385 if (pqn->q) 386 dev = pqn->q->device; 387 if (WARN_ON(!dev)) 388 return -ENODEV; 389 390 pdd = kfd_get_process_device_data(dev, pqm->process); 391 if (!pdd) { 392 pr_err("Process device data doesn't exist\n"); 393 return -1; 394 } 395 396 if (pqn->kq) { 397 /* destroy kernel queue (DIQ) */ 398 dqm = pqn->kq->dev->dqm; 399 dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd); 400 kernel_queue_uninit(pqn->kq, false); 401 } 402 403 if (pqn->q) { 404 kfd_procfs_del_queue(pqn->q); 405 dqm = pqn->q->device->dqm; 406 retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q); 407 if (retval) { 408 pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n", 409 pqm->process->pasid, 410 pqn->q->properties.queue_id, retval); 411 if (retval != -ETIME) 412 goto err_destroy_queue; 413 } 414 415 if (pqn->q->gws) { 416 amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info, 417 pqn->q->gws); 418 pdd->qpd.num_gws = 0; 419 } 420 421 uninit_queue(pqn->q); 422 } 423 424 list_del(&pqn->process_queue_list); 425 kfree(pqn); 426 clear_bit(qid, pqm->queue_slot_bitmap); 427 428 if (list_empty(&pdd->qpd.queues_list) && 429 list_empty(&pdd->qpd.priv_queue_list)) 430 dqm->ops.unregister_process(dqm, &pdd->qpd); 431 432 err_destroy_queue: 433 return retval; 434 } 435 436 int pqm_update_queue_properties(struct process_queue_manager *pqm, 437 unsigned int qid, struct queue_properties *p) 438 { 439 int retval; 440 struct process_queue_node *pqn; 441 442 pqn = get_queue_by_qid(pqm, qid); 443 if (!pqn) { 444 pr_debug("No queue %d exists for update operation\n", qid); 445 return -EFAULT; 446 } 447 448 pqn->q->properties.queue_address = p->queue_address; 449 pqn->q->properties.queue_size = p->queue_size; 450 pqn->q->properties.queue_percent = p->queue_percent; 451 pqn->q->properties.priority = p->priority; 452 453 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, 454 pqn->q, NULL); 455 if (retval != 0) 456 return retval; 457 458 return 0; 459 } 460 461 int pqm_update_mqd(struct process_queue_manager *pqm, 462 unsigned int qid, struct mqd_update_info *minfo) 463 { 464 int retval; 465 struct process_queue_node *pqn; 466 467 pqn = get_queue_by_qid(pqm, qid); 468 if (!pqn) { 469 pr_debug("No queue %d exists for update operation\n", qid); 470 return -EFAULT; 471 } 472 473 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, 474 pqn->q, minfo); 475 if (retval != 0) 476 return retval; 477 478 return 0; 479 } 480 481 struct kernel_queue *pqm_get_kernel_queue( 482 struct process_queue_manager *pqm, 483 unsigned int qid) 484 { 485 struct process_queue_node *pqn; 486 487 pqn = get_queue_by_qid(pqm, qid); 488 if (pqn && pqn->kq) 489 return pqn->kq; 490 491 return NULL; 492 } 493 494 struct queue *pqm_get_user_queue(struct process_queue_manager *pqm, 495 unsigned int qid) 496 { 497 struct process_queue_node *pqn; 498 499 pqn = get_queue_by_qid(pqm, qid); 500 return pqn ? pqn->q : NULL; 501 } 502 503 int pqm_get_wave_state(struct process_queue_manager *pqm, 504 unsigned int qid, 505 void __user *ctl_stack, 506 u32 *ctl_stack_used_size, 507 u32 *save_area_used_size) 508 { 509 struct process_queue_node *pqn; 510 511 pqn = get_queue_by_qid(pqm, qid); 512 if (!pqn) { 513 pr_debug("amdkfd: No queue %d exists for operation\n", 514 qid); 515 return -EFAULT; 516 } 517 518 return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm, 519 pqn->q, 520 ctl_stack, 521 ctl_stack_used_size, 522 save_area_used_size); 523 } 524 525 static int get_queue_data_sizes(struct kfd_process_device *pdd, 526 struct queue *q, 527 uint32_t *mqd_size, 528 uint32_t *ctl_stack_size) 529 { 530 int ret; 531 532 ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm, 533 q->properties.queue_id, 534 mqd_size, 535 ctl_stack_size); 536 if (ret) 537 pr_err("Failed to get queue dump info (%d)\n", ret); 538 539 return ret; 540 } 541 542 int kfd_process_get_queue_info(struct kfd_process *p, 543 uint32_t *num_queues, 544 uint64_t *priv_data_sizes) 545 { 546 uint32_t extra_data_sizes = 0; 547 struct queue *q; 548 int i; 549 int ret; 550 551 *num_queues = 0; 552 553 /* Run over all PDDs of the process */ 554 for (i = 0; i < p->n_pdds; i++) { 555 struct kfd_process_device *pdd = p->pdds[i]; 556 557 list_for_each_entry(q, &pdd->qpd.queues_list, list) { 558 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 559 q->properties.type == KFD_QUEUE_TYPE_SDMA || 560 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 561 uint32_t mqd_size, ctl_stack_size; 562 563 *num_queues = *num_queues + 1; 564 565 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size); 566 if (ret) 567 return ret; 568 569 extra_data_sizes += mqd_size + ctl_stack_size; 570 } else { 571 pr_err("Unsupported queue type (%d)\n", q->properties.type); 572 return -EOPNOTSUPP; 573 } 574 } 575 } 576 *priv_data_sizes = extra_data_sizes + 577 (*num_queues * sizeof(struct kfd_criu_queue_priv_data)); 578 579 return 0; 580 } 581 582 static int pqm_checkpoint_mqd(struct process_queue_manager *pqm, 583 unsigned int qid, 584 void *mqd, 585 void *ctl_stack) 586 { 587 struct process_queue_node *pqn; 588 589 pqn = get_queue_by_qid(pqm, qid); 590 if (!pqn) { 591 pr_debug("amdkfd: No queue %d exists for operation\n", qid); 592 return -EFAULT; 593 } 594 595 if (!pqn->q->device->dqm->ops.checkpoint_mqd) { 596 pr_err("amdkfd: queue dumping not supported on this device\n"); 597 return -EOPNOTSUPP; 598 } 599 600 return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm, 601 pqn->q, mqd, ctl_stack); 602 } 603 604 static int criu_checkpoint_queue(struct kfd_process_device *pdd, 605 struct queue *q, 606 struct kfd_criu_queue_priv_data *q_data) 607 { 608 uint8_t *mqd, *ctl_stack; 609 int ret; 610 611 mqd = (void *)(q_data + 1); 612 ctl_stack = mqd + q_data->mqd_size; 613 614 q_data->gpu_id = pdd->user_gpu_id; 615 q_data->type = q->properties.type; 616 q_data->format = q->properties.format; 617 q_data->q_id = q->properties.queue_id; 618 q_data->q_address = q->properties.queue_address; 619 q_data->q_size = q->properties.queue_size; 620 q_data->priority = q->properties.priority; 621 q_data->q_percent = q->properties.queue_percent; 622 q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr; 623 q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr; 624 q_data->doorbell_id = q->doorbell_id; 625 626 q_data->sdma_id = q->sdma_id; 627 628 q_data->eop_ring_buffer_address = 629 q->properties.eop_ring_buffer_address; 630 631 q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size; 632 633 q_data->ctx_save_restore_area_address = 634 q->properties.ctx_save_restore_area_address; 635 636 q_data->ctx_save_restore_area_size = 637 q->properties.ctx_save_restore_area_size; 638 639 q_data->gws = !!q->gws; 640 641 ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack); 642 if (ret) { 643 pr_err("Failed checkpoint queue_mqd (%d)\n", ret); 644 return ret; 645 } 646 647 pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id); 648 return ret; 649 } 650 651 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd, 652 uint8_t __user *user_priv, 653 unsigned int *q_index, 654 uint64_t *queues_priv_data_offset) 655 { 656 unsigned int q_private_data_size = 0; 657 uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */ 658 struct queue *q; 659 int ret = 0; 660 661 list_for_each_entry(q, &pdd->qpd.queues_list, list) { 662 struct kfd_criu_queue_priv_data *q_data; 663 uint64_t q_data_size; 664 uint32_t mqd_size; 665 uint32_t ctl_stack_size; 666 667 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE && 668 q->properties.type != KFD_QUEUE_TYPE_SDMA && 669 q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) { 670 671 pr_err("Unsupported queue type (%d)\n", q->properties.type); 672 ret = -EOPNOTSUPP; 673 break; 674 } 675 676 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size); 677 if (ret) 678 break; 679 680 q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size; 681 682 /* Increase local buffer space if needed */ 683 if (q_private_data_size < q_data_size) { 684 kfree(q_private_data); 685 686 q_private_data = kzalloc(q_data_size, GFP_KERNEL); 687 if (!q_private_data) { 688 ret = -ENOMEM; 689 break; 690 } 691 q_private_data_size = q_data_size; 692 } 693 694 q_data = (struct kfd_criu_queue_priv_data *)q_private_data; 695 696 /* data stored in this order: priv_data, mqd, ctl_stack */ 697 q_data->mqd_size = mqd_size; 698 q_data->ctl_stack_size = ctl_stack_size; 699 700 ret = criu_checkpoint_queue(pdd, q, q_data); 701 if (ret) 702 break; 703 704 q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE; 705 706 ret = copy_to_user(user_priv + *queues_priv_data_offset, 707 q_data, q_data_size); 708 if (ret) { 709 ret = -EFAULT; 710 break; 711 } 712 *queues_priv_data_offset += q_data_size; 713 *q_index = *q_index + 1; 714 } 715 716 kfree(q_private_data); 717 718 return ret; 719 } 720 721 int kfd_criu_checkpoint_queues(struct kfd_process *p, 722 uint8_t __user *user_priv_data, 723 uint64_t *priv_data_offset) 724 { 725 int ret = 0, pdd_index, q_index = 0; 726 727 for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) { 728 struct kfd_process_device *pdd = p->pdds[pdd_index]; 729 730 /* 731 * criu_checkpoint_queues_device will copy data to user and update q_index and 732 * queues_priv_data_offset 733 */ 734 ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index, 735 priv_data_offset); 736 737 if (ret) 738 break; 739 } 740 741 return ret; 742 } 743 744 static void set_queue_properties_from_criu(struct queue_properties *qp, 745 struct kfd_criu_queue_priv_data *q_data) 746 { 747 qp->is_interop = false; 748 qp->queue_percent = q_data->q_percent; 749 qp->priority = q_data->priority; 750 qp->queue_address = q_data->q_address; 751 qp->queue_size = q_data->q_size; 752 qp->read_ptr = (uint32_t *) q_data->read_ptr_addr; 753 qp->write_ptr = (uint32_t *) q_data->write_ptr_addr; 754 qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address; 755 qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size; 756 qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address; 757 qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size; 758 qp->ctl_stack_size = q_data->ctl_stack_size; 759 qp->type = q_data->type; 760 qp->format = q_data->format; 761 } 762 763 int kfd_criu_restore_queue(struct kfd_process *p, 764 uint8_t __user *user_priv_ptr, 765 uint64_t *priv_data_offset, 766 uint64_t max_priv_data_size) 767 { 768 uint8_t *mqd, *ctl_stack, *q_extra_data = NULL; 769 struct kfd_criu_queue_priv_data *q_data; 770 struct kfd_process_device *pdd; 771 uint64_t q_extra_data_size; 772 struct queue_properties qp; 773 unsigned int queue_id; 774 int ret = 0; 775 776 if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size) 777 return -EINVAL; 778 779 q_data = kmalloc(sizeof(*q_data), GFP_KERNEL); 780 if (!q_data) 781 return -ENOMEM; 782 783 ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data)); 784 if (ret) { 785 ret = -EFAULT; 786 goto exit; 787 } 788 789 *priv_data_offset += sizeof(*q_data); 790 q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size; 791 792 if (*priv_data_offset + q_extra_data_size > max_priv_data_size) { 793 ret = -EINVAL; 794 goto exit; 795 } 796 797 q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL); 798 if (!q_extra_data) { 799 ret = -ENOMEM; 800 goto exit; 801 } 802 803 ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size); 804 if (ret) { 805 ret = -EFAULT; 806 goto exit; 807 } 808 809 *priv_data_offset += q_extra_data_size; 810 811 pdd = kfd_process_device_data_by_id(p, q_data->gpu_id); 812 if (!pdd) { 813 pr_err("Failed to get pdd\n"); 814 ret = -EINVAL; 815 goto exit; 816 } 817 /* data stored in this order: mqd, ctl_stack */ 818 mqd = q_extra_data; 819 ctl_stack = mqd + q_data->mqd_size; 820 821 memset(&qp, 0, sizeof(qp)); 822 set_queue_properties_from_criu(&qp, q_data); 823 824 print_queue_properties(&qp); 825 826 ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, ctl_stack, 827 NULL); 828 if (ret) { 829 pr_err("Failed to create new queue err:%d\n", ret); 830 goto exit; 831 } 832 833 if (q_data->gws) 834 ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws); 835 836 exit: 837 if (ret) 838 pr_err("Failed to restore queue (%d)\n", ret); 839 else 840 pr_debug("Queue id %d was restored successfully\n", queue_id); 841 842 kfree(q_data); 843 844 return ret; 845 } 846 847 int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm, 848 unsigned int qid, 849 uint32_t *mqd_size, 850 uint32_t *ctl_stack_size) 851 { 852 struct process_queue_node *pqn; 853 854 pqn = get_queue_by_qid(pqm, qid); 855 if (!pqn) { 856 pr_debug("amdkfd: No queue %d exists for operation\n", qid); 857 return -EFAULT; 858 } 859 860 if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) { 861 pr_err("amdkfd: queue dumping not supported on this device\n"); 862 return -EOPNOTSUPP; 863 } 864 865 pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm, 866 pqn->q, mqd_size, 867 ctl_stack_size); 868 return 0; 869 } 870 871 #if defined(CONFIG_DEBUG_FS) 872 873 int pqm_debugfs_mqds(struct seq_file *m, void *data) 874 { 875 struct process_queue_manager *pqm = data; 876 struct process_queue_node *pqn; 877 struct queue *q; 878 enum KFD_MQD_TYPE mqd_type; 879 struct mqd_manager *mqd_mgr; 880 int r = 0; 881 882 list_for_each_entry(pqn, &pqm->queues, process_queue_list) { 883 if (pqn->q) { 884 q = pqn->q; 885 switch (q->properties.type) { 886 case KFD_QUEUE_TYPE_SDMA: 887 case KFD_QUEUE_TYPE_SDMA_XGMI: 888 seq_printf(m, " SDMA queue on device %x\n", 889 q->device->id); 890 mqd_type = KFD_MQD_TYPE_SDMA; 891 break; 892 case KFD_QUEUE_TYPE_COMPUTE: 893 seq_printf(m, " Compute queue on device %x\n", 894 q->device->id); 895 mqd_type = KFD_MQD_TYPE_CP; 896 break; 897 default: 898 seq_printf(m, 899 " Bad user queue type %d on device %x\n", 900 q->properties.type, q->device->id); 901 continue; 902 } 903 mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type]; 904 } else if (pqn->kq) { 905 q = pqn->kq->queue; 906 mqd_mgr = pqn->kq->mqd_mgr; 907 switch (q->properties.type) { 908 case KFD_QUEUE_TYPE_DIQ: 909 seq_printf(m, " DIQ on device %x\n", 910 pqn->kq->dev->id); 911 break; 912 default: 913 seq_printf(m, 914 " Bad kernel queue type %d on device %x\n", 915 q->properties.type, 916 pqn->kq->dev->id); 917 continue; 918 } 919 } else { 920 seq_printf(m, 921 " Weird: Queue node with neither kernel nor user queue\n"); 922 continue; 923 } 924 925 r = mqd_mgr->debugfs_show_mqd(m, q->mqd); 926 if (r != 0) 927 break; 928 } 929 930 return r; 931 } 932 933 #endif 934