1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/slab.h> 25 #include <linux/list.h> 26 #include <linux/types.h> 27 #include <linux/printk.h> 28 #include <linux/bitops.h> 29 #include <linux/sched.h> 30 #include "kfd_priv.h" 31 #include "kfd_device_queue_manager.h" 32 #include "kfd_mqd_manager.h" 33 #include "cik_regs.h" 34 #include "kfd_kernel_queue.h" 35 36 /* Size of the per-pipe EOP queue */ 37 #define CIK_HPD_EOP_BYTES_LOG2 11 38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 39 40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 41 unsigned int pasid, unsigned int vmid); 42 43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 44 struct queue *q, 45 struct qcm_process_device *qpd); 46 47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); 48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); 49 50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 51 struct queue *q, 52 struct qcm_process_device *qpd); 53 54 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 55 unsigned int sdma_queue_id); 56 57 static inline 58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 59 { 60 if (type == KFD_QUEUE_TYPE_SDMA) 61 return KFD_MQD_TYPE_SDMA; 62 return KFD_MQD_TYPE_CP; 63 } 64 65 unsigned int get_first_pipe(struct device_queue_manager *dqm) 66 { 67 BUG_ON(!dqm || !dqm->dev); 68 return dqm->dev->shared_resources.first_compute_pipe; 69 } 70 71 unsigned int get_pipes_num(struct device_queue_manager *dqm) 72 { 73 BUG_ON(!dqm || !dqm->dev); 74 return dqm->dev->shared_resources.compute_pipe_count; 75 } 76 77 static inline unsigned int get_pipes_num_cpsch(void) 78 { 79 return PIPE_PER_ME_CP_SCHEDULING; 80 } 81 82 void program_sh_mem_settings(struct device_queue_manager *dqm, 83 struct qcm_process_device *qpd) 84 { 85 return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, 86 qpd->sh_mem_config, 87 qpd->sh_mem_ape1_base, 88 qpd->sh_mem_ape1_limit, 89 qpd->sh_mem_bases); 90 } 91 92 static int allocate_vmid(struct device_queue_manager *dqm, 93 struct qcm_process_device *qpd, 94 struct queue *q) 95 { 96 int bit, allocated_vmid; 97 98 if (dqm->vmid_bitmap == 0) 99 return -ENOMEM; 100 101 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); 102 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 103 104 /* Kaveri kfd vmid's starts from vmid 8 */ 105 allocated_vmid = bit + KFD_VMID_START_OFFSET; 106 pr_debug("kfd: vmid allocation %d\n", allocated_vmid); 107 qpd->vmid = allocated_vmid; 108 q->properties.vmid = allocated_vmid; 109 110 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); 111 program_sh_mem_settings(dqm, qpd); 112 113 return 0; 114 } 115 116 static void deallocate_vmid(struct device_queue_manager *dqm, 117 struct qcm_process_device *qpd, 118 struct queue *q) 119 { 120 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 121 122 /* Release the vmid mapping */ 123 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 124 125 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 126 qpd->vmid = 0; 127 q->properties.vmid = 0; 128 } 129 130 static int create_queue_nocpsch(struct device_queue_manager *dqm, 131 struct queue *q, 132 struct qcm_process_device *qpd, 133 int *allocated_vmid) 134 { 135 int retval; 136 137 BUG_ON(!dqm || !q || !qpd || !allocated_vmid); 138 139 pr_debug("kfd: In func %s\n", __func__); 140 print_queue(q); 141 142 mutex_lock(&dqm->lock); 143 144 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 145 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 146 dqm->total_queue_count); 147 mutex_unlock(&dqm->lock); 148 return -EPERM; 149 } 150 151 if (list_empty(&qpd->queues_list)) { 152 retval = allocate_vmid(dqm, qpd, q); 153 if (retval != 0) { 154 mutex_unlock(&dqm->lock); 155 return retval; 156 } 157 } 158 *allocated_vmid = qpd->vmid; 159 q->properties.vmid = qpd->vmid; 160 161 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 162 retval = create_compute_queue_nocpsch(dqm, q, qpd); 163 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 164 retval = create_sdma_queue_nocpsch(dqm, q, qpd); 165 166 if (retval != 0) { 167 if (list_empty(&qpd->queues_list)) { 168 deallocate_vmid(dqm, qpd, q); 169 *allocated_vmid = 0; 170 } 171 mutex_unlock(&dqm->lock); 172 return retval; 173 } 174 175 list_add(&q->list, &qpd->queues_list); 176 if (q->properties.is_active) 177 dqm->queue_count++; 178 179 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 180 dqm->sdma_queue_count++; 181 182 /* 183 * Unconditionally increment this counter, regardless of the queue's 184 * type or whether the queue is active. 185 */ 186 dqm->total_queue_count++; 187 pr_debug("Total of %d queues are accountable so far\n", 188 dqm->total_queue_count); 189 190 mutex_unlock(&dqm->lock); 191 return 0; 192 } 193 194 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 195 { 196 bool set; 197 int pipe, bit, i; 198 199 set = false; 200 201 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm); 202 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) { 203 if (dqm->allocated_queues[pipe] != 0) { 204 bit = find_first_bit( 205 (unsigned long *)&dqm->allocated_queues[pipe], 206 QUEUES_PER_PIPE); 207 208 clear_bit(bit, 209 (unsigned long *)&dqm->allocated_queues[pipe]); 210 q->pipe = pipe; 211 q->queue = bit; 212 set = true; 213 break; 214 } 215 } 216 217 if (set == false) 218 return -EBUSY; 219 220 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", 221 __func__, q->pipe, q->queue); 222 /* horizontal hqd allocation */ 223 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); 224 225 return 0; 226 } 227 228 static inline void deallocate_hqd(struct device_queue_manager *dqm, 229 struct queue *q) 230 { 231 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); 232 } 233 234 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 235 struct queue *q, 236 struct qcm_process_device *qpd) 237 { 238 int retval; 239 struct mqd_manager *mqd; 240 241 BUG_ON(!dqm || !q || !qpd); 242 243 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 244 if (mqd == NULL) 245 return -ENOMEM; 246 247 retval = allocate_hqd(dqm, q); 248 if (retval != 0) 249 return retval; 250 251 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 252 &q->gart_mqd_addr, &q->properties); 253 if (retval != 0) { 254 deallocate_hqd(dqm, q); 255 return retval; 256 } 257 258 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", 259 q->pipe, 260 q->queue); 261 262 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 263 q->queue, (uint32_t __user *) q->properties.write_ptr); 264 if (retval != 0) { 265 deallocate_hqd(dqm, q); 266 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 267 return retval; 268 } 269 270 return 0; 271 } 272 273 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 274 struct qcm_process_device *qpd, 275 struct queue *q) 276 { 277 int retval; 278 struct mqd_manager *mqd; 279 280 BUG_ON(!dqm || !q || !q->mqd || !qpd); 281 282 retval = 0; 283 284 pr_debug("kfd: In Func %s\n", __func__); 285 286 mutex_lock(&dqm->lock); 287 288 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 289 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 290 if (mqd == NULL) { 291 retval = -ENOMEM; 292 goto out; 293 } 294 deallocate_hqd(dqm, q); 295 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 296 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 297 if (mqd == NULL) { 298 retval = -ENOMEM; 299 goto out; 300 } 301 dqm->sdma_queue_count--; 302 deallocate_sdma_queue(dqm, q->sdma_id); 303 } else { 304 pr_debug("q->properties.type is invalid (%d)\n", 305 q->properties.type); 306 retval = -EINVAL; 307 goto out; 308 } 309 310 retval = mqd->destroy_mqd(mqd, q->mqd, 311 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 312 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, 313 q->pipe, q->queue); 314 315 if (retval != 0) 316 goto out; 317 318 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 319 320 list_del(&q->list); 321 if (list_empty(&qpd->queues_list)) 322 deallocate_vmid(dqm, qpd, q); 323 if (q->properties.is_active) 324 dqm->queue_count--; 325 326 /* 327 * Unconditionally decrement this counter, regardless of the queue's 328 * type 329 */ 330 dqm->total_queue_count--; 331 pr_debug("Total of %d queues are accountable so far\n", 332 dqm->total_queue_count); 333 334 out: 335 mutex_unlock(&dqm->lock); 336 return retval; 337 } 338 339 static int update_queue(struct device_queue_manager *dqm, struct queue *q) 340 { 341 int retval; 342 struct mqd_manager *mqd; 343 bool prev_active = false; 344 345 BUG_ON(!dqm || !q || !q->mqd); 346 347 mutex_lock(&dqm->lock); 348 mqd = dqm->ops.get_mqd_manager(dqm, 349 get_mqd_type_from_queue_type(q->properties.type)); 350 if (mqd == NULL) { 351 mutex_unlock(&dqm->lock); 352 return -ENOMEM; 353 } 354 355 if (q->properties.is_active == true) 356 prev_active = true; 357 358 /* 359 * 360 * check active state vs. the previous state 361 * and modify counter accordingly 362 */ 363 retval = mqd->update_mqd(mqd, q->mqd, &q->properties); 364 if ((q->properties.is_active == true) && (prev_active == false)) 365 dqm->queue_count++; 366 else if ((q->properties.is_active == false) && (prev_active == true)) 367 dqm->queue_count--; 368 369 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 370 retval = execute_queues_cpsch(dqm, false); 371 372 mutex_unlock(&dqm->lock); 373 return retval; 374 } 375 376 static struct mqd_manager *get_mqd_manager_nocpsch( 377 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) 378 { 379 struct mqd_manager *mqd; 380 381 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); 382 383 pr_debug("kfd: In func %s mqd type %d\n", __func__, type); 384 385 mqd = dqm->mqds[type]; 386 if (!mqd) { 387 mqd = mqd_manager_init(type, dqm->dev); 388 if (mqd == NULL) 389 pr_err("kfd: mqd manager is NULL"); 390 dqm->mqds[type] = mqd; 391 } 392 393 return mqd; 394 } 395 396 static int register_process_nocpsch(struct device_queue_manager *dqm, 397 struct qcm_process_device *qpd) 398 { 399 struct device_process_node *n; 400 int retval; 401 402 BUG_ON(!dqm || !qpd); 403 404 pr_debug("kfd: In func %s\n", __func__); 405 406 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); 407 if (!n) 408 return -ENOMEM; 409 410 n->qpd = qpd; 411 412 mutex_lock(&dqm->lock); 413 list_add(&n->list, &dqm->queues); 414 415 retval = dqm->ops_asic_specific.register_process(dqm, qpd); 416 417 dqm->processes_count++; 418 419 mutex_unlock(&dqm->lock); 420 421 return retval; 422 } 423 424 static int unregister_process_nocpsch(struct device_queue_manager *dqm, 425 struct qcm_process_device *qpd) 426 { 427 int retval; 428 struct device_process_node *cur, *next; 429 430 BUG_ON(!dqm || !qpd); 431 432 BUG_ON(!list_empty(&qpd->queues_list)); 433 434 pr_debug("kfd: In func %s\n", __func__); 435 436 retval = 0; 437 mutex_lock(&dqm->lock); 438 439 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 440 if (qpd == cur->qpd) { 441 list_del(&cur->list); 442 kfree(cur); 443 dqm->processes_count--; 444 goto out; 445 } 446 } 447 /* qpd not found in dqm list */ 448 retval = 1; 449 out: 450 mutex_unlock(&dqm->lock); 451 return retval; 452 } 453 454 static int 455 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, 456 unsigned int vmid) 457 { 458 uint32_t pasid_mapping; 459 460 pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 461 ATC_VMID_PASID_MAPPING_VALID; 462 return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, 463 vmid); 464 } 465 466 int init_pipelines(struct device_queue_manager *dqm, 467 unsigned int pipes_num, unsigned int first_pipe) 468 { 469 void *hpdptr; 470 struct mqd_manager *mqd; 471 unsigned int i, err, inx; 472 uint64_t pipe_hpd_addr; 473 474 BUG_ON(!dqm || !dqm->dev); 475 476 pr_debug("kfd: In func %s\n", __func__); 477 478 /* 479 * Allocate memory for the HPDs. This is hardware-owned per-pipe data. 480 * The driver never accesses this memory after zeroing it. 481 * It doesn't even have to be saved/restored on suspend/resume 482 * because it contains no data when there are no active queues. 483 */ 484 485 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num, 486 &dqm->pipeline_mem); 487 488 if (err) { 489 pr_err("kfd: error allocate vidmem num pipes: %d\n", 490 pipes_num); 491 return -ENOMEM; 492 } 493 494 hpdptr = dqm->pipeline_mem->cpu_ptr; 495 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; 496 497 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); 498 499 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 500 if (mqd == NULL) { 501 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 502 return -ENOMEM; 503 } 504 505 for (i = 0; i < pipes_num; i++) { 506 inx = i + first_pipe; 507 /* 508 * HPD buffer on GTT is allocated by amdkfd, no need to waste 509 * space in GTT for pipelines we don't initialize 510 */ 511 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; 512 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); 513 /* = log2(bytes/4)-1 */ 514 kfd2kgd->init_pipeline(dqm->dev->kgd, inx, 515 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); 516 } 517 518 return 0; 519 } 520 521 static int init_scheduler(struct device_queue_manager *dqm) 522 { 523 int retval; 524 525 BUG_ON(!dqm); 526 527 pr_debug("kfd: In %s\n", __func__); 528 529 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm)); 530 return retval; 531 } 532 533 static int initialize_nocpsch(struct device_queue_manager *dqm) 534 { 535 int i; 536 537 BUG_ON(!dqm); 538 539 pr_debug("kfd: In func %s num of pipes: %d\n", 540 __func__, get_pipes_num(dqm)); 541 542 mutex_init(&dqm->lock); 543 INIT_LIST_HEAD(&dqm->queues); 544 dqm->queue_count = dqm->next_pipe_to_allocate = 0; 545 dqm->sdma_queue_count = 0; 546 dqm->allocated_queues = kcalloc(get_pipes_num(dqm), 547 sizeof(unsigned int), GFP_KERNEL); 548 if (!dqm->allocated_queues) { 549 mutex_destroy(&dqm->lock); 550 return -ENOMEM; 551 } 552 553 for (i = 0; i < get_pipes_num(dqm); i++) 554 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; 555 556 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; 557 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1; 558 559 init_scheduler(dqm); 560 return 0; 561 } 562 563 static void uninitialize_nocpsch(struct device_queue_manager *dqm) 564 { 565 int i; 566 567 BUG_ON(!dqm); 568 569 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); 570 571 kfree(dqm->allocated_queues); 572 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 573 kfree(dqm->mqds[i]); 574 mutex_destroy(&dqm->lock); 575 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 576 } 577 578 static int start_nocpsch(struct device_queue_manager *dqm) 579 { 580 return 0; 581 } 582 583 static int stop_nocpsch(struct device_queue_manager *dqm) 584 { 585 return 0; 586 } 587 588 static int allocate_sdma_queue(struct device_queue_manager *dqm, 589 unsigned int *sdma_queue_id) 590 { 591 int bit; 592 593 if (dqm->sdma_bitmap == 0) 594 return -ENOMEM; 595 596 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap, 597 CIK_SDMA_QUEUES); 598 599 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap); 600 *sdma_queue_id = bit; 601 602 return 0; 603 } 604 605 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 606 unsigned int sdma_queue_id) 607 { 608 if (sdma_queue_id >= CIK_SDMA_QUEUES) 609 return; 610 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap); 611 } 612 613 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, 614 struct qcm_process_device *qpd) 615 { 616 uint32_t value = SDMA_ATC; 617 618 if (q->process->is_32bit_user_mode) 619 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); 620 else 621 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( 622 qpd_to_pdd(qpd))); 623 q->properties.sdma_vm_addr = value; 624 } 625 626 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 627 struct queue *q, 628 struct qcm_process_device *qpd) 629 { 630 struct mqd_manager *mqd; 631 int retval; 632 633 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 634 if (!mqd) 635 return -ENOMEM; 636 637 retval = allocate_sdma_queue(dqm, &q->sdma_id); 638 if (retval != 0) 639 return retval; 640 641 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE; 642 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM; 643 644 pr_debug("kfd: sdma id is: %d\n", q->sdma_id); 645 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id); 646 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id); 647 648 init_sdma_vm(dqm, q, qpd); 649 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 650 &q->gart_mqd_addr, &q->properties); 651 if (retval != 0) { 652 deallocate_sdma_queue(dqm, q->sdma_id); 653 return retval; 654 } 655 656 retval = mqd->load_mqd(mqd, q->mqd, 0, 657 0, NULL); 658 if (retval != 0) { 659 deallocate_sdma_queue(dqm, q->sdma_id); 660 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 661 return retval; 662 } 663 664 return 0; 665 } 666 667 /* 668 * Device Queue Manager implementation for cp scheduler 669 */ 670 671 static int set_sched_resources(struct device_queue_manager *dqm) 672 { 673 struct scheduling_resources res; 674 unsigned int queue_num, queue_mask; 675 676 BUG_ON(!dqm); 677 678 pr_debug("kfd: In func %s\n", __func__); 679 680 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; 681 queue_mask = (1 << queue_num) - 1; 682 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; 683 res.vmid_mask <<= KFD_VMID_START_OFFSET; 684 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); 685 res.gws_mask = res.oac_mask = res.gds_heap_base = 686 res.gds_heap_size = 0; 687 688 pr_debug("kfd: scheduling resources:\n" 689 " vmid mask: 0x%8X\n" 690 " queue mask: 0x%8llX\n", 691 res.vmid_mask, res.queue_mask); 692 693 return pm_send_set_resources(&dqm->packets, &res); 694 } 695 696 static int initialize_cpsch(struct device_queue_manager *dqm) 697 { 698 int retval; 699 700 BUG_ON(!dqm); 701 702 pr_debug("kfd: In func %s num of pipes: %d\n", 703 __func__, get_pipes_num_cpsch()); 704 705 mutex_init(&dqm->lock); 706 INIT_LIST_HEAD(&dqm->queues); 707 dqm->queue_count = dqm->processes_count = 0; 708 dqm->sdma_queue_count = 0; 709 dqm->active_runlist = false; 710 retval = dqm->ops_asic_specific.initialize(dqm); 711 if (retval != 0) 712 goto fail_init_pipelines; 713 714 return 0; 715 716 fail_init_pipelines: 717 mutex_destroy(&dqm->lock); 718 return retval; 719 } 720 721 static int start_cpsch(struct device_queue_manager *dqm) 722 { 723 struct device_process_node *node; 724 int retval; 725 726 BUG_ON(!dqm); 727 728 retval = 0; 729 730 retval = pm_init(&dqm->packets, dqm); 731 if (retval != 0) 732 goto fail_packet_manager_init; 733 734 retval = set_sched_resources(dqm); 735 if (retval != 0) 736 goto fail_set_sched_resources; 737 738 pr_debug("kfd: allocating fence memory\n"); 739 740 /* allocate fence memory on the gart */ 741 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 742 &dqm->fence_mem); 743 744 if (retval != 0) 745 goto fail_allocate_vidmem; 746 747 dqm->fence_addr = dqm->fence_mem->cpu_ptr; 748 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 749 list_for_each_entry(node, &dqm->queues, list) 750 if (node->qpd->pqm->process && dqm->dev) 751 kfd_bind_process_to_device(dqm->dev, 752 node->qpd->pqm->process); 753 754 execute_queues_cpsch(dqm, true); 755 756 return 0; 757 fail_allocate_vidmem: 758 fail_set_sched_resources: 759 pm_uninit(&dqm->packets); 760 fail_packet_manager_init: 761 return retval; 762 } 763 764 static int stop_cpsch(struct device_queue_manager *dqm) 765 { 766 struct device_process_node *node; 767 struct kfd_process_device *pdd; 768 769 BUG_ON(!dqm); 770 771 destroy_queues_cpsch(dqm, true); 772 773 list_for_each_entry(node, &dqm->queues, list) { 774 pdd = qpd_to_pdd(node->qpd); 775 pdd->bound = false; 776 } 777 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 778 pm_uninit(&dqm->packets); 779 780 return 0; 781 } 782 783 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 784 struct kernel_queue *kq, 785 struct qcm_process_device *qpd) 786 { 787 BUG_ON(!dqm || !kq || !qpd); 788 789 pr_debug("kfd: In func %s\n", __func__); 790 791 mutex_lock(&dqm->lock); 792 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 793 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n", 794 dqm->total_queue_count); 795 mutex_unlock(&dqm->lock); 796 return -EPERM; 797 } 798 799 /* 800 * Unconditionally increment this counter, regardless of the queue's 801 * type or whether the queue is active. 802 */ 803 dqm->total_queue_count++; 804 pr_debug("Total of %d queues are accountable so far\n", 805 dqm->total_queue_count); 806 807 list_add(&kq->list, &qpd->priv_queue_list); 808 dqm->queue_count++; 809 qpd->is_debug = true; 810 execute_queues_cpsch(dqm, false); 811 mutex_unlock(&dqm->lock); 812 813 return 0; 814 } 815 816 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 817 struct kernel_queue *kq, 818 struct qcm_process_device *qpd) 819 { 820 BUG_ON(!dqm || !kq); 821 822 pr_debug("kfd: In %s\n", __func__); 823 824 mutex_lock(&dqm->lock); 825 destroy_queues_cpsch(dqm, false); 826 list_del(&kq->list); 827 dqm->queue_count--; 828 qpd->is_debug = false; 829 execute_queues_cpsch(dqm, false); 830 /* 831 * Unconditionally decrement this counter, regardless of the queue's 832 * type. 833 */ 834 dqm->total_queue_count--; 835 pr_debug("Total of %d queues are accountable so far\n", 836 dqm->total_queue_count); 837 mutex_unlock(&dqm->lock); 838 } 839 840 static void select_sdma_engine_id(struct queue *q) 841 { 842 static int sdma_id; 843 844 q->sdma_id = sdma_id; 845 sdma_id = (sdma_id + 1) % 2; 846 } 847 848 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 849 struct qcm_process_device *qpd, int *allocate_vmid) 850 { 851 int retval; 852 struct mqd_manager *mqd; 853 854 BUG_ON(!dqm || !q || !qpd); 855 856 retval = 0; 857 858 if (allocate_vmid) 859 *allocate_vmid = 0; 860 861 mutex_lock(&dqm->lock); 862 863 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 864 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 865 dqm->total_queue_count); 866 retval = -EPERM; 867 goto out; 868 } 869 870 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 871 select_sdma_engine_id(q); 872 873 mqd = dqm->ops.get_mqd_manager(dqm, 874 get_mqd_type_from_queue_type(q->properties.type)); 875 876 if (mqd == NULL) { 877 mutex_unlock(&dqm->lock); 878 return -ENOMEM; 879 } 880 881 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 882 &q->gart_mqd_addr, &q->properties); 883 if (retval != 0) 884 goto out; 885 886 list_add(&q->list, &qpd->queues_list); 887 if (q->properties.is_active) { 888 dqm->queue_count++; 889 retval = execute_queues_cpsch(dqm, false); 890 } 891 892 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 893 dqm->sdma_queue_count++; 894 /* 895 * Unconditionally increment this counter, regardless of the queue's 896 * type or whether the queue is active. 897 */ 898 dqm->total_queue_count++; 899 900 pr_debug("Total of %d queues are accountable so far\n", 901 dqm->total_queue_count); 902 903 out: 904 mutex_unlock(&dqm->lock); 905 return retval; 906 } 907 908 static int fence_wait_timeout(unsigned int *fence_addr, 909 unsigned int fence_value, 910 unsigned long timeout) 911 { 912 BUG_ON(!fence_addr); 913 timeout += jiffies; 914 915 while (*fence_addr != fence_value) { 916 if (time_after(jiffies, timeout)) { 917 pr_err("kfd: qcm fence wait loop timeout expired\n"); 918 return -ETIME; 919 } 920 schedule(); 921 } 922 923 return 0; 924 } 925 926 static int destroy_sdma_queues(struct device_queue_manager *dqm, 927 unsigned int sdma_engine) 928 { 929 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, 930 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 931 sdma_engine); 932 } 933 934 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) 935 { 936 int retval; 937 938 BUG_ON(!dqm); 939 940 retval = 0; 941 942 if (lock) 943 mutex_lock(&dqm->lock); 944 if (dqm->active_runlist == false) 945 goto out; 946 947 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n", 948 dqm->sdma_queue_count); 949 950 if (dqm->sdma_queue_count > 0) { 951 destroy_sdma_queues(dqm, 0); 952 destroy_sdma_queues(dqm, 1); 953 } 954 955 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, 956 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); 957 if (retval != 0) 958 goto out; 959 960 *dqm->fence_addr = KFD_FENCE_INIT; 961 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, 962 KFD_FENCE_COMPLETED); 963 /* should be timed out */ 964 fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, 965 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); 966 pm_release_ib(&dqm->packets); 967 dqm->active_runlist = false; 968 969 out: 970 if (lock) 971 mutex_unlock(&dqm->lock); 972 return retval; 973 } 974 975 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) 976 { 977 int retval; 978 979 BUG_ON(!dqm); 980 981 if (lock) 982 mutex_lock(&dqm->lock); 983 984 retval = destroy_queues_cpsch(dqm, false); 985 if (retval != 0) { 986 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); 987 goto out; 988 } 989 990 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { 991 retval = 0; 992 goto out; 993 } 994 995 if (dqm->active_runlist) { 996 retval = 0; 997 goto out; 998 } 999 1000 retval = pm_send_runlist(&dqm->packets, &dqm->queues); 1001 if (retval != 0) { 1002 pr_err("kfd: failed to execute runlist"); 1003 goto out; 1004 } 1005 dqm->active_runlist = true; 1006 1007 out: 1008 if (lock) 1009 mutex_unlock(&dqm->lock); 1010 return retval; 1011 } 1012 1013 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 1014 struct qcm_process_device *qpd, 1015 struct queue *q) 1016 { 1017 int retval; 1018 struct mqd_manager *mqd; 1019 1020 BUG_ON(!dqm || !qpd || !q); 1021 1022 retval = 0; 1023 1024 /* remove queue from list to prevent rescheduling after preemption */ 1025 mutex_lock(&dqm->lock); 1026 mqd = dqm->ops.get_mqd_manager(dqm, 1027 get_mqd_type_from_queue_type(q->properties.type)); 1028 if (!mqd) { 1029 retval = -ENOMEM; 1030 goto failed; 1031 } 1032 1033 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1034 dqm->sdma_queue_count--; 1035 1036 list_del(&q->list); 1037 if (q->properties.is_active) 1038 dqm->queue_count--; 1039 1040 execute_queues_cpsch(dqm, false); 1041 1042 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 1043 1044 /* 1045 * Unconditionally decrement this counter, regardless of the queue's 1046 * type 1047 */ 1048 dqm->total_queue_count--; 1049 pr_debug("Total of %d queues are accountable so far\n", 1050 dqm->total_queue_count); 1051 1052 mutex_unlock(&dqm->lock); 1053 1054 return 0; 1055 1056 failed: 1057 mutex_unlock(&dqm->lock); 1058 return retval; 1059 } 1060 1061 /* 1062 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 1063 * stay in user mode. 1064 */ 1065 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 1066 /* APE1 limit is inclusive and 64K aligned. */ 1067 #define APE1_LIMIT_ALIGNMENT 0xFFFF 1068 1069 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 1070 struct qcm_process_device *qpd, 1071 enum cache_policy default_policy, 1072 enum cache_policy alternate_policy, 1073 void __user *alternate_aperture_base, 1074 uint64_t alternate_aperture_size) 1075 { 1076 bool retval; 1077 1078 pr_debug("kfd: In func %s\n", __func__); 1079 1080 mutex_lock(&dqm->lock); 1081 1082 if (alternate_aperture_size == 0) { 1083 /* base > limit disables APE1 */ 1084 qpd->sh_mem_ape1_base = 1; 1085 qpd->sh_mem_ape1_limit = 0; 1086 } else { 1087 /* 1088 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 1089 * SH_MEM_APE1_BASE[31:0], 0x0000 } 1090 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 1091 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 1092 * Verify that the base and size parameters can be 1093 * represented in this format and convert them. 1094 * Additionally restrict APE1 to user-mode addresses. 1095 */ 1096 1097 uint64_t base = (uintptr_t)alternate_aperture_base; 1098 uint64_t limit = base + alternate_aperture_size - 1; 1099 1100 if (limit <= base) 1101 goto out; 1102 1103 if ((base & APE1_FIXED_BITS_MASK) != 0) 1104 goto out; 1105 1106 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) 1107 goto out; 1108 1109 qpd->sh_mem_ape1_base = base >> 16; 1110 qpd->sh_mem_ape1_limit = limit >> 16; 1111 } 1112 1113 retval = dqm->ops_asic_specific.set_cache_memory_policy( 1114 dqm, 1115 qpd, 1116 default_policy, 1117 alternate_policy, 1118 alternate_aperture_base, 1119 alternate_aperture_size); 1120 1121 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 1122 program_sh_mem_settings(dqm, qpd); 1123 1124 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 1125 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 1126 qpd->sh_mem_ape1_limit); 1127 1128 mutex_unlock(&dqm->lock); 1129 return retval; 1130 1131 out: 1132 mutex_unlock(&dqm->lock); 1133 return false; 1134 } 1135 1136 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) 1137 { 1138 struct device_queue_manager *dqm; 1139 1140 BUG_ON(!dev); 1141 1142 pr_debug("kfd: loading device queue manager\n"); 1143 1144 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); 1145 if (!dqm) 1146 return NULL; 1147 1148 dqm->dev = dev; 1149 switch (sched_policy) { 1150 case KFD_SCHED_POLICY_HWS: 1151 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 1152 /* initialize dqm for cp scheduling */ 1153 dqm->ops.create_queue = create_queue_cpsch; 1154 dqm->ops.initialize = initialize_cpsch; 1155 dqm->ops.start = start_cpsch; 1156 dqm->ops.stop = stop_cpsch; 1157 dqm->ops.destroy_queue = destroy_queue_cpsch; 1158 dqm->ops.update_queue = update_queue; 1159 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1160 dqm->ops.register_process = register_process_nocpsch; 1161 dqm->ops.unregister_process = unregister_process_nocpsch; 1162 dqm->ops.uninitialize = uninitialize_nocpsch; 1163 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 1164 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 1165 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1166 break; 1167 case KFD_SCHED_POLICY_NO_HWS: 1168 /* initialize dqm for no cp scheduling */ 1169 dqm->ops.start = start_nocpsch; 1170 dqm->ops.stop = stop_nocpsch; 1171 dqm->ops.create_queue = create_queue_nocpsch; 1172 dqm->ops.destroy_queue = destroy_queue_nocpsch; 1173 dqm->ops.update_queue = update_queue; 1174 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1175 dqm->ops.register_process = register_process_nocpsch; 1176 dqm->ops.unregister_process = unregister_process_nocpsch; 1177 dqm->ops.initialize = initialize_nocpsch; 1178 dqm->ops.uninitialize = uninitialize_nocpsch; 1179 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1180 break; 1181 default: 1182 BUG(); 1183 break; 1184 } 1185 1186 switch (dev->device_info->asic_family) { 1187 case CHIP_CARRIZO: 1188 device_queue_manager_init_vi(&dqm->ops_asic_specific); 1189 break; 1190 1191 case CHIP_KAVERI: 1192 device_queue_manager_init_cik(&dqm->ops_asic_specific); 1193 break; 1194 } 1195 1196 if (dqm->ops.initialize(dqm) != 0) { 1197 kfree(dqm); 1198 return NULL; 1199 } 1200 1201 return dqm; 1202 } 1203 1204 void device_queue_manager_uninit(struct device_queue_manager *dqm) 1205 { 1206 BUG_ON(!dqm); 1207 1208 dqm->ops.uninitialize(dqm); 1209 kfree(dqm); 1210 } 1211