1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/slab.h> 25 #include <linux/list.h> 26 #include <linux/types.h> 27 #include <linux/printk.h> 28 #include <linux/bitops.h> 29 #include <linux/sched.h> 30 #include "kfd_priv.h" 31 #include "kfd_device_queue_manager.h" 32 #include "kfd_mqd_manager.h" 33 #include "cik_regs.h" 34 #include "kfd_kernel_queue.h" 35 36 /* Size of the per-pipe EOP queue */ 37 #define CIK_HPD_EOP_BYTES_LOG2 11 38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 39 40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 41 unsigned int pasid, unsigned int vmid); 42 43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 44 struct queue *q, 45 struct qcm_process_device *qpd); 46 47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); 48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); 49 50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 51 struct queue *q, 52 struct qcm_process_device *qpd); 53 54 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 55 unsigned int sdma_queue_id); 56 57 static inline 58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 59 { 60 if (type == KFD_QUEUE_TYPE_SDMA) 61 return KFD_MQD_TYPE_SDMA; 62 return KFD_MQD_TYPE_CP; 63 } 64 65 unsigned int get_first_pipe(struct device_queue_manager *dqm) 66 { 67 BUG_ON(!dqm || !dqm->dev); 68 return dqm->dev->shared_resources.first_compute_pipe; 69 } 70 71 unsigned int get_pipes_num(struct device_queue_manager *dqm) 72 { 73 BUG_ON(!dqm || !dqm->dev); 74 return dqm->dev->shared_resources.compute_pipe_count; 75 } 76 77 static inline unsigned int get_pipes_num_cpsch(void) 78 { 79 return PIPE_PER_ME_CP_SCHEDULING; 80 } 81 82 void program_sh_mem_settings(struct device_queue_manager *dqm, 83 struct qcm_process_device *qpd) 84 { 85 return dqm->dev->kfd2kgd->program_sh_mem_settings( 86 dqm->dev->kgd, qpd->vmid, 87 qpd->sh_mem_config, 88 qpd->sh_mem_ape1_base, 89 qpd->sh_mem_ape1_limit, 90 qpd->sh_mem_bases); 91 } 92 93 static int allocate_vmid(struct device_queue_manager *dqm, 94 struct qcm_process_device *qpd, 95 struct queue *q) 96 { 97 int bit, allocated_vmid; 98 99 if (dqm->vmid_bitmap == 0) 100 return -ENOMEM; 101 102 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); 103 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 104 105 /* Kaveri kfd vmid's starts from vmid 8 */ 106 allocated_vmid = bit + KFD_VMID_START_OFFSET; 107 pr_debug("kfd: vmid allocation %d\n", allocated_vmid); 108 qpd->vmid = allocated_vmid; 109 q->properties.vmid = allocated_vmid; 110 111 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); 112 program_sh_mem_settings(dqm, qpd); 113 114 return 0; 115 } 116 117 static void deallocate_vmid(struct device_queue_manager *dqm, 118 struct qcm_process_device *qpd, 119 struct queue *q) 120 { 121 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 122 123 /* Release the vmid mapping */ 124 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 125 126 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 127 qpd->vmid = 0; 128 q->properties.vmid = 0; 129 } 130 131 static int create_queue_nocpsch(struct device_queue_manager *dqm, 132 struct queue *q, 133 struct qcm_process_device *qpd, 134 int *allocated_vmid) 135 { 136 int retval; 137 138 BUG_ON(!dqm || !q || !qpd || !allocated_vmid); 139 140 pr_debug("kfd: In func %s\n", __func__); 141 print_queue(q); 142 143 mutex_lock(&dqm->lock); 144 145 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 146 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 147 dqm->total_queue_count); 148 mutex_unlock(&dqm->lock); 149 return -EPERM; 150 } 151 152 if (list_empty(&qpd->queues_list)) { 153 retval = allocate_vmid(dqm, qpd, q); 154 if (retval != 0) { 155 mutex_unlock(&dqm->lock); 156 return retval; 157 } 158 } 159 *allocated_vmid = qpd->vmid; 160 q->properties.vmid = qpd->vmid; 161 162 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 163 retval = create_compute_queue_nocpsch(dqm, q, qpd); 164 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 165 retval = create_sdma_queue_nocpsch(dqm, q, qpd); 166 167 if (retval != 0) { 168 if (list_empty(&qpd->queues_list)) { 169 deallocate_vmid(dqm, qpd, q); 170 *allocated_vmid = 0; 171 } 172 mutex_unlock(&dqm->lock); 173 return retval; 174 } 175 176 list_add(&q->list, &qpd->queues_list); 177 if (q->properties.is_active) 178 dqm->queue_count++; 179 180 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 181 dqm->sdma_queue_count++; 182 183 /* 184 * Unconditionally increment this counter, regardless of the queue's 185 * type or whether the queue is active. 186 */ 187 dqm->total_queue_count++; 188 pr_debug("Total of %d queues are accountable so far\n", 189 dqm->total_queue_count); 190 191 mutex_unlock(&dqm->lock); 192 return 0; 193 } 194 195 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 196 { 197 bool set; 198 int pipe, bit, i; 199 200 set = false; 201 202 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm); 203 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) { 204 if (dqm->allocated_queues[pipe] != 0) { 205 bit = find_first_bit( 206 (unsigned long *)&dqm->allocated_queues[pipe], 207 QUEUES_PER_PIPE); 208 209 clear_bit(bit, 210 (unsigned long *)&dqm->allocated_queues[pipe]); 211 q->pipe = pipe; 212 q->queue = bit; 213 set = true; 214 break; 215 } 216 } 217 218 if (set == false) 219 return -EBUSY; 220 221 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", 222 __func__, q->pipe, q->queue); 223 /* horizontal hqd allocation */ 224 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); 225 226 return 0; 227 } 228 229 static inline void deallocate_hqd(struct device_queue_manager *dqm, 230 struct queue *q) 231 { 232 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); 233 } 234 235 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 236 struct queue *q, 237 struct qcm_process_device *qpd) 238 { 239 int retval; 240 struct mqd_manager *mqd; 241 242 BUG_ON(!dqm || !q || !qpd); 243 244 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 245 if (mqd == NULL) 246 return -ENOMEM; 247 248 retval = allocate_hqd(dqm, q); 249 if (retval != 0) 250 return retval; 251 252 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 253 &q->gart_mqd_addr, &q->properties); 254 if (retval != 0) { 255 deallocate_hqd(dqm, q); 256 return retval; 257 } 258 259 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", 260 q->pipe, 261 q->queue); 262 263 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 264 q->queue, (uint32_t __user *) q->properties.write_ptr); 265 if (retval != 0) { 266 deallocate_hqd(dqm, q); 267 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 268 return retval; 269 } 270 271 return 0; 272 } 273 274 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 275 struct qcm_process_device *qpd, 276 struct queue *q) 277 { 278 int retval; 279 struct mqd_manager *mqd; 280 281 BUG_ON(!dqm || !q || !q->mqd || !qpd); 282 283 retval = 0; 284 285 pr_debug("kfd: In Func %s\n", __func__); 286 287 mutex_lock(&dqm->lock); 288 289 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 290 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 291 if (mqd == NULL) { 292 retval = -ENOMEM; 293 goto out; 294 } 295 deallocate_hqd(dqm, q); 296 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 297 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 298 if (mqd == NULL) { 299 retval = -ENOMEM; 300 goto out; 301 } 302 dqm->sdma_queue_count--; 303 deallocate_sdma_queue(dqm, q->sdma_id); 304 } else { 305 pr_debug("q->properties.type is invalid (%d)\n", 306 q->properties.type); 307 retval = -EINVAL; 308 goto out; 309 } 310 311 retval = mqd->destroy_mqd(mqd, q->mqd, 312 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 313 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, 314 q->pipe, q->queue); 315 316 if (retval != 0) 317 goto out; 318 319 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 320 321 list_del(&q->list); 322 if (list_empty(&qpd->queues_list)) 323 deallocate_vmid(dqm, qpd, q); 324 if (q->properties.is_active) 325 dqm->queue_count--; 326 327 /* 328 * Unconditionally decrement this counter, regardless of the queue's 329 * type 330 */ 331 dqm->total_queue_count--; 332 pr_debug("Total of %d queues are accountable so far\n", 333 dqm->total_queue_count); 334 335 out: 336 mutex_unlock(&dqm->lock); 337 return retval; 338 } 339 340 static int update_queue(struct device_queue_manager *dqm, struct queue *q) 341 { 342 int retval; 343 struct mqd_manager *mqd; 344 bool prev_active = false; 345 346 BUG_ON(!dqm || !q || !q->mqd); 347 348 mutex_lock(&dqm->lock); 349 mqd = dqm->ops.get_mqd_manager(dqm, 350 get_mqd_type_from_queue_type(q->properties.type)); 351 if (mqd == NULL) { 352 mutex_unlock(&dqm->lock); 353 return -ENOMEM; 354 } 355 356 if (q->properties.is_active == true) 357 prev_active = true; 358 359 /* 360 * 361 * check active state vs. the previous state 362 * and modify counter accordingly 363 */ 364 retval = mqd->update_mqd(mqd, q->mqd, &q->properties); 365 if ((q->properties.is_active == true) && (prev_active == false)) 366 dqm->queue_count++; 367 else if ((q->properties.is_active == false) && (prev_active == true)) 368 dqm->queue_count--; 369 370 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 371 retval = execute_queues_cpsch(dqm, false); 372 373 mutex_unlock(&dqm->lock); 374 return retval; 375 } 376 377 static struct mqd_manager *get_mqd_manager_nocpsch( 378 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) 379 { 380 struct mqd_manager *mqd; 381 382 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); 383 384 pr_debug("kfd: In func %s mqd type %d\n", __func__, type); 385 386 mqd = dqm->mqds[type]; 387 if (!mqd) { 388 mqd = mqd_manager_init(type, dqm->dev); 389 if (mqd == NULL) 390 pr_err("kfd: mqd manager is NULL"); 391 dqm->mqds[type] = mqd; 392 } 393 394 return mqd; 395 } 396 397 static int register_process_nocpsch(struct device_queue_manager *dqm, 398 struct qcm_process_device *qpd) 399 { 400 struct device_process_node *n; 401 int retval; 402 403 BUG_ON(!dqm || !qpd); 404 405 pr_debug("kfd: In func %s\n", __func__); 406 407 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); 408 if (!n) 409 return -ENOMEM; 410 411 n->qpd = qpd; 412 413 mutex_lock(&dqm->lock); 414 list_add(&n->list, &dqm->queues); 415 416 retval = dqm->ops_asic_specific.register_process(dqm, qpd); 417 418 dqm->processes_count++; 419 420 mutex_unlock(&dqm->lock); 421 422 return retval; 423 } 424 425 static int unregister_process_nocpsch(struct device_queue_manager *dqm, 426 struct qcm_process_device *qpd) 427 { 428 int retval; 429 struct device_process_node *cur, *next; 430 431 BUG_ON(!dqm || !qpd); 432 433 BUG_ON(!list_empty(&qpd->queues_list)); 434 435 pr_debug("kfd: In func %s\n", __func__); 436 437 retval = 0; 438 mutex_lock(&dqm->lock); 439 440 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 441 if (qpd == cur->qpd) { 442 list_del(&cur->list); 443 kfree(cur); 444 dqm->processes_count--; 445 goto out; 446 } 447 } 448 /* qpd not found in dqm list */ 449 retval = 1; 450 out: 451 mutex_unlock(&dqm->lock); 452 return retval; 453 } 454 455 static int 456 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, 457 unsigned int vmid) 458 { 459 uint32_t pasid_mapping; 460 461 pasid_mapping = (pasid == 0) ? 0 : 462 (uint32_t)pasid | 463 ATC_VMID_PASID_MAPPING_VALID; 464 465 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 466 dqm->dev->kgd, pasid_mapping, 467 vmid); 468 } 469 470 int init_pipelines(struct device_queue_manager *dqm, 471 unsigned int pipes_num, unsigned int first_pipe) 472 { 473 void *hpdptr; 474 struct mqd_manager *mqd; 475 unsigned int i, err, inx; 476 uint64_t pipe_hpd_addr; 477 478 BUG_ON(!dqm || !dqm->dev); 479 480 pr_debug("kfd: In func %s\n", __func__); 481 482 /* 483 * Allocate memory for the HPDs. This is hardware-owned per-pipe data. 484 * The driver never accesses this memory after zeroing it. 485 * It doesn't even have to be saved/restored on suspend/resume 486 * because it contains no data when there are no active queues. 487 */ 488 489 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num, 490 &dqm->pipeline_mem); 491 492 if (err) { 493 pr_err("kfd: error allocate vidmem num pipes: %d\n", 494 pipes_num); 495 return -ENOMEM; 496 } 497 498 hpdptr = dqm->pipeline_mem->cpu_ptr; 499 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; 500 501 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); 502 503 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 504 if (mqd == NULL) { 505 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 506 return -ENOMEM; 507 } 508 509 for (i = 0; i < pipes_num; i++) { 510 inx = i + first_pipe; 511 /* 512 * HPD buffer on GTT is allocated by amdkfd, no need to waste 513 * space in GTT for pipelines we don't initialize 514 */ 515 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; 516 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); 517 /* = log2(bytes/4)-1 */ 518 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx, 519 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); 520 } 521 522 return 0; 523 } 524 525 static int init_scheduler(struct device_queue_manager *dqm) 526 { 527 int retval; 528 529 BUG_ON(!dqm); 530 531 pr_debug("kfd: In %s\n", __func__); 532 533 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm)); 534 return retval; 535 } 536 537 static int initialize_nocpsch(struct device_queue_manager *dqm) 538 { 539 int i; 540 541 BUG_ON(!dqm); 542 543 pr_debug("kfd: In func %s num of pipes: %d\n", 544 __func__, get_pipes_num(dqm)); 545 546 mutex_init(&dqm->lock); 547 INIT_LIST_HEAD(&dqm->queues); 548 dqm->queue_count = dqm->next_pipe_to_allocate = 0; 549 dqm->sdma_queue_count = 0; 550 dqm->allocated_queues = kcalloc(get_pipes_num(dqm), 551 sizeof(unsigned int), GFP_KERNEL); 552 if (!dqm->allocated_queues) { 553 mutex_destroy(&dqm->lock); 554 return -ENOMEM; 555 } 556 557 for (i = 0; i < get_pipes_num(dqm); i++) 558 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; 559 560 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; 561 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1; 562 563 init_scheduler(dqm); 564 return 0; 565 } 566 567 static void uninitialize_nocpsch(struct device_queue_manager *dqm) 568 { 569 int i; 570 571 BUG_ON(!dqm); 572 573 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); 574 575 kfree(dqm->allocated_queues); 576 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 577 kfree(dqm->mqds[i]); 578 mutex_destroy(&dqm->lock); 579 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 580 } 581 582 static int start_nocpsch(struct device_queue_manager *dqm) 583 { 584 return 0; 585 } 586 587 static int stop_nocpsch(struct device_queue_manager *dqm) 588 { 589 return 0; 590 } 591 592 static int allocate_sdma_queue(struct device_queue_manager *dqm, 593 unsigned int *sdma_queue_id) 594 { 595 int bit; 596 597 if (dqm->sdma_bitmap == 0) 598 return -ENOMEM; 599 600 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap, 601 CIK_SDMA_QUEUES); 602 603 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap); 604 *sdma_queue_id = bit; 605 606 return 0; 607 } 608 609 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 610 unsigned int sdma_queue_id) 611 { 612 if (sdma_queue_id >= CIK_SDMA_QUEUES) 613 return; 614 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap); 615 } 616 617 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, 618 struct qcm_process_device *qpd) 619 { 620 uint32_t value = SDMA_ATC; 621 622 if (q->process->is_32bit_user_mode) 623 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); 624 else 625 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( 626 qpd_to_pdd(qpd))); 627 q->properties.sdma_vm_addr = value; 628 } 629 630 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 631 struct queue *q, 632 struct qcm_process_device *qpd) 633 { 634 struct mqd_manager *mqd; 635 int retval; 636 637 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 638 if (!mqd) 639 return -ENOMEM; 640 641 retval = allocate_sdma_queue(dqm, &q->sdma_id); 642 if (retval != 0) 643 return retval; 644 645 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE; 646 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM; 647 648 pr_debug("kfd: sdma id is: %d\n", q->sdma_id); 649 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id); 650 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id); 651 652 init_sdma_vm(dqm, q, qpd); 653 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 654 &q->gart_mqd_addr, &q->properties); 655 if (retval != 0) { 656 deallocate_sdma_queue(dqm, q->sdma_id); 657 return retval; 658 } 659 660 retval = mqd->load_mqd(mqd, q->mqd, 0, 661 0, NULL); 662 if (retval != 0) { 663 deallocate_sdma_queue(dqm, q->sdma_id); 664 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 665 return retval; 666 } 667 668 return 0; 669 } 670 671 /* 672 * Device Queue Manager implementation for cp scheduler 673 */ 674 675 static int set_sched_resources(struct device_queue_manager *dqm) 676 { 677 struct scheduling_resources res; 678 unsigned int queue_num, queue_mask; 679 680 BUG_ON(!dqm); 681 682 pr_debug("kfd: In func %s\n", __func__); 683 684 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; 685 queue_mask = (1 << queue_num) - 1; 686 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; 687 res.vmid_mask <<= KFD_VMID_START_OFFSET; 688 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); 689 res.gws_mask = res.oac_mask = res.gds_heap_base = 690 res.gds_heap_size = 0; 691 692 pr_debug("kfd: scheduling resources:\n" 693 " vmid mask: 0x%8X\n" 694 " queue mask: 0x%8llX\n", 695 res.vmid_mask, res.queue_mask); 696 697 return pm_send_set_resources(&dqm->packets, &res); 698 } 699 700 static int initialize_cpsch(struct device_queue_manager *dqm) 701 { 702 int retval; 703 704 BUG_ON(!dqm); 705 706 pr_debug("kfd: In func %s num of pipes: %d\n", 707 __func__, get_pipes_num_cpsch()); 708 709 mutex_init(&dqm->lock); 710 INIT_LIST_HEAD(&dqm->queues); 711 dqm->queue_count = dqm->processes_count = 0; 712 dqm->sdma_queue_count = 0; 713 dqm->active_runlist = false; 714 retval = dqm->ops_asic_specific.initialize(dqm); 715 if (retval != 0) 716 goto fail_init_pipelines; 717 718 return 0; 719 720 fail_init_pipelines: 721 mutex_destroy(&dqm->lock); 722 return retval; 723 } 724 725 static int start_cpsch(struct device_queue_manager *dqm) 726 { 727 struct device_process_node *node; 728 int retval; 729 730 BUG_ON(!dqm); 731 732 retval = 0; 733 734 retval = pm_init(&dqm->packets, dqm); 735 if (retval != 0) 736 goto fail_packet_manager_init; 737 738 retval = set_sched_resources(dqm); 739 if (retval != 0) 740 goto fail_set_sched_resources; 741 742 pr_debug("kfd: allocating fence memory\n"); 743 744 /* allocate fence memory on the gart */ 745 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 746 &dqm->fence_mem); 747 748 if (retval != 0) 749 goto fail_allocate_vidmem; 750 751 dqm->fence_addr = dqm->fence_mem->cpu_ptr; 752 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 753 list_for_each_entry(node, &dqm->queues, list) 754 if (node->qpd->pqm->process && dqm->dev) 755 kfd_bind_process_to_device(dqm->dev, 756 node->qpd->pqm->process); 757 758 execute_queues_cpsch(dqm, true); 759 760 return 0; 761 fail_allocate_vidmem: 762 fail_set_sched_resources: 763 pm_uninit(&dqm->packets); 764 fail_packet_manager_init: 765 return retval; 766 } 767 768 static int stop_cpsch(struct device_queue_manager *dqm) 769 { 770 struct device_process_node *node; 771 struct kfd_process_device *pdd; 772 773 BUG_ON(!dqm); 774 775 destroy_queues_cpsch(dqm, true); 776 777 list_for_each_entry(node, &dqm->queues, list) { 778 pdd = qpd_to_pdd(node->qpd); 779 pdd->bound = false; 780 } 781 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 782 pm_uninit(&dqm->packets); 783 784 return 0; 785 } 786 787 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 788 struct kernel_queue *kq, 789 struct qcm_process_device *qpd) 790 { 791 BUG_ON(!dqm || !kq || !qpd); 792 793 pr_debug("kfd: In func %s\n", __func__); 794 795 mutex_lock(&dqm->lock); 796 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 797 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n", 798 dqm->total_queue_count); 799 mutex_unlock(&dqm->lock); 800 return -EPERM; 801 } 802 803 /* 804 * Unconditionally increment this counter, regardless of the queue's 805 * type or whether the queue is active. 806 */ 807 dqm->total_queue_count++; 808 pr_debug("Total of %d queues are accountable so far\n", 809 dqm->total_queue_count); 810 811 list_add(&kq->list, &qpd->priv_queue_list); 812 dqm->queue_count++; 813 qpd->is_debug = true; 814 execute_queues_cpsch(dqm, false); 815 mutex_unlock(&dqm->lock); 816 817 return 0; 818 } 819 820 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 821 struct kernel_queue *kq, 822 struct qcm_process_device *qpd) 823 { 824 BUG_ON(!dqm || !kq); 825 826 pr_debug("kfd: In %s\n", __func__); 827 828 mutex_lock(&dqm->lock); 829 destroy_queues_cpsch(dqm, false); 830 list_del(&kq->list); 831 dqm->queue_count--; 832 qpd->is_debug = false; 833 execute_queues_cpsch(dqm, false); 834 /* 835 * Unconditionally decrement this counter, regardless of the queue's 836 * type. 837 */ 838 dqm->total_queue_count--; 839 pr_debug("Total of %d queues are accountable so far\n", 840 dqm->total_queue_count); 841 mutex_unlock(&dqm->lock); 842 } 843 844 static void select_sdma_engine_id(struct queue *q) 845 { 846 static int sdma_id; 847 848 q->sdma_id = sdma_id; 849 sdma_id = (sdma_id + 1) % 2; 850 } 851 852 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 853 struct qcm_process_device *qpd, int *allocate_vmid) 854 { 855 int retval; 856 struct mqd_manager *mqd; 857 858 BUG_ON(!dqm || !q || !qpd); 859 860 retval = 0; 861 862 if (allocate_vmid) 863 *allocate_vmid = 0; 864 865 mutex_lock(&dqm->lock); 866 867 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 868 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 869 dqm->total_queue_count); 870 retval = -EPERM; 871 goto out; 872 } 873 874 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 875 select_sdma_engine_id(q); 876 877 mqd = dqm->ops.get_mqd_manager(dqm, 878 get_mqd_type_from_queue_type(q->properties.type)); 879 880 if (mqd == NULL) { 881 mutex_unlock(&dqm->lock); 882 return -ENOMEM; 883 } 884 885 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 886 &q->gart_mqd_addr, &q->properties); 887 if (retval != 0) 888 goto out; 889 890 list_add(&q->list, &qpd->queues_list); 891 if (q->properties.is_active) { 892 dqm->queue_count++; 893 retval = execute_queues_cpsch(dqm, false); 894 } 895 896 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 897 dqm->sdma_queue_count++; 898 /* 899 * Unconditionally increment this counter, regardless of the queue's 900 * type or whether the queue is active. 901 */ 902 dqm->total_queue_count++; 903 904 pr_debug("Total of %d queues are accountable so far\n", 905 dqm->total_queue_count); 906 907 out: 908 mutex_unlock(&dqm->lock); 909 return retval; 910 } 911 912 static int amdkfd_fence_wait_timeout(unsigned int *fence_addr, 913 unsigned int fence_value, 914 unsigned long timeout) 915 { 916 BUG_ON(!fence_addr); 917 timeout += jiffies; 918 919 while (*fence_addr != fence_value) { 920 if (time_after(jiffies, timeout)) { 921 pr_err("kfd: qcm fence wait loop timeout expired\n"); 922 return -ETIME; 923 } 924 schedule(); 925 } 926 927 return 0; 928 } 929 930 static int destroy_sdma_queues(struct device_queue_manager *dqm, 931 unsigned int sdma_engine) 932 { 933 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, 934 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 935 sdma_engine); 936 } 937 938 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) 939 { 940 int retval; 941 942 BUG_ON(!dqm); 943 944 retval = 0; 945 946 if (lock) 947 mutex_lock(&dqm->lock); 948 if (dqm->active_runlist == false) 949 goto out; 950 951 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n", 952 dqm->sdma_queue_count); 953 954 if (dqm->sdma_queue_count > 0) { 955 destroy_sdma_queues(dqm, 0); 956 destroy_sdma_queues(dqm, 1); 957 } 958 959 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, 960 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); 961 if (retval != 0) 962 goto out; 963 964 *dqm->fence_addr = KFD_FENCE_INIT; 965 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, 966 KFD_FENCE_COMPLETED); 967 /* should be timed out */ 968 amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, 969 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); 970 pm_release_ib(&dqm->packets); 971 dqm->active_runlist = false; 972 973 out: 974 if (lock) 975 mutex_unlock(&dqm->lock); 976 return retval; 977 } 978 979 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) 980 { 981 int retval; 982 983 BUG_ON(!dqm); 984 985 if (lock) 986 mutex_lock(&dqm->lock); 987 988 retval = destroy_queues_cpsch(dqm, false); 989 if (retval != 0) { 990 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); 991 goto out; 992 } 993 994 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { 995 retval = 0; 996 goto out; 997 } 998 999 if (dqm->active_runlist) { 1000 retval = 0; 1001 goto out; 1002 } 1003 1004 retval = pm_send_runlist(&dqm->packets, &dqm->queues); 1005 if (retval != 0) { 1006 pr_err("kfd: failed to execute runlist"); 1007 goto out; 1008 } 1009 dqm->active_runlist = true; 1010 1011 out: 1012 if (lock) 1013 mutex_unlock(&dqm->lock); 1014 return retval; 1015 } 1016 1017 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 1018 struct qcm_process_device *qpd, 1019 struct queue *q) 1020 { 1021 int retval; 1022 struct mqd_manager *mqd; 1023 1024 BUG_ON(!dqm || !qpd || !q); 1025 1026 retval = 0; 1027 1028 /* remove queue from list to prevent rescheduling after preemption */ 1029 mutex_lock(&dqm->lock); 1030 mqd = dqm->ops.get_mqd_manager(dqm, 1031 get_mqd_type_from_queue_type(q->properties.type)); 1032 if (!mqd) { 1033 retval = -ENOMEM; 1034 goto failed; 1035 } 1036 1037 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1038 dqm->sdma_queue_count--; 1039 1040 list_del(&q->list); 1041 if (q->properties.is_active) 1042 dqm->queue_count--; 1043 1044 execute_queues_cpsch(dqm, false); 1045 1046 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 1047 1048 /* 1049 * Unconditionally decrement this counter, regardless of the queue's 1050 * type 1051 */ 1052 dqm->total_queue_count--; 1053 pr_debug("Total of %d queues are accountable so far\n", 1054 dqm->total_queue_count); 1055 1056 mutex_unlock(&dqm->lock); 1057 1058 return 0; 1059 1060 failed: 1061 mutex_unlock(&dqm->lock); 1062 return retval; 1063 } 1064 1065 /* 1066 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 1067 * stay in user mode. 1068 */ 1069 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 1070 /* APE1 limit is inclusive and 64K aligned. */ 1071 #define APE1_LIMIT_ALIGNMENT 0xFFFF 1072 1073 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 1074 struct qcm_process_device *qpd, 1075 enum cache_policy default_policy, 1076 enum cache_policy alternate_policy, 1077 void __user *alternate_aperture_base, 1078 uint64_t alternate_aperture_size) 1079 { 1080 bool retval; 1081 1082 pr_debug("kfd: In func %s\n", __func__); 1083 1084 mutex_lock(&dqm->lock); 1085 1086 if (alternate_aperture_size == 0) { 1087 /* base > limit disables APE1 */ 1088 qpd->sh_mem_ape1_base = 1; 1089 qpd->sh_mem_ape1_limit = 0; 1090 } else { 1091 /* 1092 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 1093 * SH_MEM_APE1_BASE[31:0], 0x0000 } 1094 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 1095 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 1096 * Verify that the base and size parameters can be 1097 * represented in this format and convert them. 1098 * Additionally restrict APE1 to user-mode addresses. 1099 */ 1100 1101 uint64_t base = (uintptr_t)alternate_aperture_base; 1102 uint64_t limit = base + alternate_aperture_size - 1; 1103 1104 if (limit <= base) 1105 goto out; 1106 1107 if ((base & APE1_FIXED_BITS_MASK) != 0) 1108 goto out; 1109 1110 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) 1111 goto out; 1112 1113 qpd->sh_mem_ape1_base = base >> 16; 1114 qpd->sh_mem_ape1_limit = limit >> 16; 1115 } 1116 1117 retval = dqm->ops_asic_specific.set_cache_memory_policy( 1118 dqm, 1119 qpd, 1120 default_policy, 1121 alternate_policy, 1122 alternate_aperture_base, 1123 alternate_aperture_size); 1124 1125 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 1126 program_sh_mem_settings(dqm, qpd); 1127 1128 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 1129 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 1130 qpd->sh_mem_ape1_limit); 1131 1132 mutex_unlock(&dqm->lock); 1133 return retval; 1134 1135 out: 1136 mutex_unlock(&dqm->lock); 1137 return false; 1138 } 1139 1140 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) 1141 { 1142 struct device_queue_manager *dqm; 1143 1144 BUG_ON(!dev); 1145 1146 pr_debug("kfd: loading device queue manager\n"); 1147 1148 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); 1149 if (!dqm) 1150 return NULL; 1151 1152 dqm->dev = dev; 1153 switch (sched_policy) { 1154 case KFD_SCHED_POLICY_HWS: 1155 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 1156 /* initialize dqm for cp scheduling */ 1157 dqm->ops.create_queue = create_queue_cpsch; 1158 dqm->ops.initialize = initialize_cpsch; 1159 dqm->ops.start = start_cpsch; 1160 dqm->ops.stop = stop_cpsch; 1161 dqm->ops.destroy_queue = destroy_queue_cpsch; 1162 dqm->ops.update_queue = update_queue; 1163 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1164 dqm->ops.register_process = register_process_nocpsch; 1165 dqm->ops.unregister_process = unregister_process_nocpsch; 1166 dqm->ops.uninitialize = uninitialize_nocpsch; 1167 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 1168 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 1169 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1170 break; 1171 case KFD_SCHED_POLICY_NO_HWS: 1172 /* initialize dqm for no cp scheduling */ 1173 dqm->ops.start = start_nocpsch; 1174 dqm->ops.stop = stop_nocpsch; 1175 dqm->ops.create_queue = create_queue_nocpsch; 1176 dqm->ops.destroy_queue = destroy_queue_nocpsch; 1177 dqm->ops.update_queue = update_queue; 1178 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1179 dqm->ops.register_process = register_process_nocpsch; 1180 dqm->ops.unregister_process = unregister_process_nocpsch; 1181 dqm->ops.initialize = initialize_nocpsch; 1182 dqm->ops.uninitialize = uninitialize_nocpsch; 1183 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1184 break; 1185 default: 1186 BUG(); 1187 break; 1188 } 1189 1190 switch (dev->device_info->asic_family) { 1191 case CHIP_CARRIZO: 1192 device_queue_manager_init_vi(&dqm->ops_asic_specific); 1193 break; 1194 1195 case CHIP_KAVERI: 1196 device_queue_manager_init_cik(&dqm->ops_asic_specific); 1197 break; 1198 } 1199 1200 if (dqm->ops.initialize(dqm) != 0) { 1201 kfree(dqm); 1202 return NULL; 1203 } 1204 1205 return dqm; 1206 } 1207 1208 void device_queue_manager_uninit(struct device_queue_manager *dqm) 1209 { 1210 BUG_ON(!dqm); 1211 1212 dqm->ops.uninitialize(dqm); 1213 kfree(dqm); 1214 } 1215