1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/slab.h> 25 #include <linux/list.h> 26 #include <linux/types.h> 27 #include <linux/printk.h> 28 #include <linux/bitops.h> 29 #include <linux/sched.h> 30 #include "kfd_priv.h" 31 #include "kfd_device_queue_manager.h" 32 #include "kfd_mqd_manager.h" 33 #include "cik_regs.h" 34 #include "kfd_kernel_queue.h" 35 36 /* Size of the per-pipe EOP queue */ 37 #define CIK_HPD_EOP_BYTES_LOG2 11 38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 39 40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 41 unsigned int pasid, unsigned int vmid); 42 43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 44 struct queue *q, 45 struct qcm_process_device *qpd); 46 47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); 48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); 49 50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 51 struct queue *q, 52 struct qcm_process_device *qpd); 53 54 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 55 unsigned int sdma_queue_id); 56 57 static inline 58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 59 { 60 if (type == KFD_QUEUE_TYPE_SDMA) 61 return KFD_MQD_TYPE_SDMA; 62 return KFD_MQD_TYPE_CP; 63 } 64 65 unsigned int get_first_pipe(struct device_queue_manager *dqm) 66 { 67 BUG_ON(!dqm || !dqm->dev); 68 return dqm->dev->shared_resources.first_compute_pipe; 69 } 70 71 unsigned int get_pipes_num(struct device_queue_manager *dqm) 72 { 73 BUG_ON(!dqm || !dqm->dev); 74 return dqm->dev->shared_resources.compute_pipe_count; 75 } 76 77 static inline unsigned int get_pipes_num_cpsch(void) 78 { 79 return PIPE_PER_ME_CP_SCHEDULING; 80 } 81 82 void program_sh_mem_settings(struct device_queue_manager *dqm, 83 struct qcm_process_device *qpd) 84 { 85 return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, 86 qpd->sh_mem_config, 87 qpd->sh_mem_ape1_base, 88 qpd->sh_mem_ape1_limit, 89 qpd->sh_mem_bases); 90 } 91 92 static int allocate_vmid(struct device_queue_manager *dqm, 93 struct qcm_process_device *qpd, 94 struct queue *q) 95 { 96 int bit, allocated_vmid; 97 98 if (dqm->vmid_bitmap == 0) 99 return -ENOMEM; 100 101 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); 102 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 103 104 /* Kaveri kfd vmid's starts from vmid 8 */ 105 allocated_vmid = bit + KFD_VMID_START_OFFSET; 106 pr_debug("kfd: vmid allocation %d\n", allocated_vmid); 107 qpd->vmid = allocated_vmid; 108 q->properties.vmid = allocated_vmid; 109 110 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); 111 program_sh_mem_settings(dqm, qpd); 112 113 return 0; 114 } 115 116 static void deallocate_vmid(struct device_queue_manager *dqm, 117 struct qcm_process_device *qpd, 118 struct queue *q) 119 { 120 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 121 122 /* Release the vmid mapping */ 123 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 124 125 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 126 qpd->vmid = 0; 127 q->properties.vmid = 0; 128 } 129 130 static int create_queue_nocpsch(struct device_queue_manager *dqm, 131 struct queue *q, 132 struct qcm_process_device *qpd, 133 int *allocated_vmid) 134 { 135 int retval; 136 137 BUG_ON(!dqm || !q || !qpd || !allocated_vmid); 138 139 pr_debug("kfd: In func %s\n", __func__); 140 print_queue(q); 141 142 mutex_lock(&dqm->lock); 143 144 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 145 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 146 dqm->total_queue_count); 147 mutex_unlock(&dqm->lock); 148 return -EPERM; 149 } 150 151 if (list_empty(&qpd->queues_list)) { 152 retval = allocate_vmid(dqm, qpd, q); 153 if (retval != 0) { 154 mutex_unlock(&dqm->lock); 155 return retval; 156 } 157 } 158 *allocated_vmid = qpd->vmid; 159 q->properties.vmid = qpd->vmid; 160 161 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 162 retval = create_compute_queue_nocpsch(dqm, q, qpd); 163 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 164 retval = create_sdma_queue_nocpsch(dqm, q, qpd); 165 166 if (retval != 0) { 167 if (list_empty(&qpd->queues_list)) { 168 deallocate_vmid(dqm, qpd, q); 169 *allocated_vmid = 0; 170 } 171 mutex_unlock(&dqm->lock); 172 return retval; 173 } 174 175 list_add(&q->list, &qpd->queues_list); 176 if (q->properties.is_active) 177 dqm->queue_count++; 178 179 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 180 dqm->sdma_queue_count++; 181 182 /* 183 * Unconditionally increment this counter, regardless of the queue's 184 * type or whether the queue is active. 185 */ 186 dqm->total_queue_count++; 187 pr_debug("Total of %d queues are accountable so far\n", 188 dqm->total_queue_count); 189 190 mutex_unlock(&dqm->lock); 191 return 0; 192 } 193 194 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 195 { 196 bool set; 197 int pipe, bit, i; 198 199 set = false; 200 201 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm); 202 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) { 203 if (dqm->allocated_queues[pipe] != 0) { 204 bit = find_first_bit( 205 (unsigned long *)&dqm->allocated_queues[pipe], 206 QUEUES_PER_PIPE); 207 208 clear_bit(bit, 209 (unsigned long *)&dqm->allocated_queues[pipe]); 210 q->pipe = pipe; 211 q->queue = bit; 212 set = true; 213 break; 214 } 215 } 216 217 if (set == false) 218 return -EBUSY; 219 220 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", 221 __func__, q->pipe, q->queue); 222 /* horizontal hqd allocation */ 223 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); 224 225 return 0; 226 } 227 228 static inline void deallocate_hqd(struct device_queue_manager *dqm, 229 struct queue *q) 230 { 231 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); 232 } 233 234 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 235 struct queue *q, 236 struct qcm_process_device *qpd) 237 { 238 int retval; 239 struct mqd_manager *mqd; 240 241 BUG_ON(!dqm || !q || !qpd); 242 243 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 244 if (mqd == NULL) 245 return -ENOMEM; 246 247 retval = allocate_hqd(dqm, q); 248 if (retval != 0) 249 return retval; 250 251 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 252 &q->gart_mqd_addr, &q->properties); 253 if (retval != 0) { 254 deallocate_hqd(dqm, q); 255 return retval; 256 } 257 258 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", 259 q->pipe, 260 q->queue); 261 262 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 263 q->queue, (uint32_t __user *) q->properties.write_ptr); 264 if (retval != 0) { 265 deallocate_hqd(dqm, q); 266 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 267 return retval; 268 } 269 270 return 0; 271 } 272 273 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 274 struct qcm_process_device *qpd, 275 struct queue *q) 276 { 277 int retval; 278 struct mqd_manager *mqd; 279 280 BUG_ON(!dqm || !q || !q->mqd || !qpd); 281 282 retval = 0; 283 284 pr_debug("kfd: In Func %s\n", __func__); 285 286 mutex_lock(&dqm->lock); 287 288 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 289 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 290 if (mqd == NULL) { 291 retval = -ENOMEM; 292 goto out; 293 } 294 deallocate_hqd(dqm, q); 295 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 296 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 297 if (mqd == NULL) { 298 retval = -ENOMEM; 299 goto out; 300 } 301 dqm->sdma_queue_count--; 302 deallocate_sdma_queue(dqm, q->sdma_id); 303 } else { 304 pr_debug("q->properties.type is invalid (%d)\n", 305 q->properties.type); 306 retval = -EINVAL; 307 goto out; 308 } 309 310 retval = mqd->destroy_mqd(mqd, q->mqd, 311 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 312 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, 313 q->pipe, q->queue); 314 315 if (retval != 0) 316 goto out; 317 318 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 319 320 list_del(&q->list); 321 if (list_empty(&qpd->queues_list)) 322 deallocate_vmid(dqm, qpd, q); 323 if (q->properties.is_active) 324 dqm->queue_count--; 325 326 /* 327 * Unconditionally decrement this counter, regardless of the queue's 328 * type 329 */ 330 dqm->total_queue_count--; 331 pr_debug("Total of %d queues are accountable so far\n", 332 dqm->total_queue_count); 333 334 out: 335 mutex_unlock(&dqm->lock); 336 return retval; 337 } 338 339 static int update_queue(struct device_queue_manager *dqm, struct queue *q) 340 { 341 int retval; 342 struct mqd_manager *mqd; 343 bool prev_active = false; 344 345 BUG_ON(!dqm || !q || !q->mqd); 346 347 mutex_lock(&dqm->lock); 348 mqd = dqm->ops.get_mqd_manager(dqm, 349 get_mqd_type_from_queue_type(q->properties.type)); 350 if (mqd == NULL) { 351 mutex_unlock(&dqm->lock); 352 return -ENOMEM; 353 } 354 355 if (q->properties.is_active == true) 356 prev_active = true; 357 358 /* 359 * 360 * check active state vs. the previous state 361 * and modify counter accordingly 362 */ 363 retval = mqd->update_mqd(mqd, q->mqd, &q->properties); 364 if ((q->properties.is_active == true) && (prev_active == false)) 365 dqm->queue_count++; 366 else if ((q->properties.is_active == false) && (prev_active == true)) 367 dqm->queue_count--; 368 369 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 370 retval = execute_queues_cpsch(dqm, false); 371 372 mutex_unlock(&dqm->lock); 373 return retval; 374 } 375 376 static struct mqd_manager *get_mqd_manager_nocpsch( 377 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) 378 { 379 struct mqd_manager *mqd; 380 381 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); 382 383 pr_debug("kfd: In func %s mqd type %d\n", __func__, type); 384 385 mqd = dqm->mqds[type]; 386 if (!mqd) { 387 mqd = mqd_manager_init(type, dqm->dev); 388 if (mqd == NULL) 389 pr_err("kfd: mqd manager is NULL"); 390 dqm->mqds[type] = mqd; 391 } 392 393 return mqd; 394 } 395 396 static int register_process_nocpsch(struct device_queue_manager *dqm, 397 struct qcm_process_device *qpd) 398 { 399 struct device_process_node *n; 400 int retval; 401 402 BUG_ON(!dqm || !qpd); 403 404 pr_debug("kfd: In func %s\n", __func__); 405 406 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); 407 if (!n) 408 return -ENOMEM; 409 410 n->qpd = qpd; 411 412 mutex_lock(&dqm->lock); 413 list_add(&n->list, &dqm->queues); 414 415 retval = dqm->ops_asic_specific.register_process(dqm, qpd); 416 417 dqm->processes_count++; 418 419 mutex_unlock(&dqm->lock); 420 421 return retval; 422 } 423 424 static int unregister_process_nocpsch(struct device_queue_manager *dqm, 425 struct qcm_process_device *qpd) 426 { 427 int retval; 428 struct device_process_node *cur, *next; 429 430 BUG_ON(!dqm || !qpd); 431 432 BUG_ON(!list_empty(&qpd->queues_list)); 433 434 pr_debug("kfd: In func %s\n", __func__); 435 436 retval = 0; 437 mutex_lock(&dqm->lock); 438 439 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 440 if (qpd == cur->qpd) { 441 list_del(&cur->list); 442 kfree(cur); 443 dqm->processes_count--; 444 goto out; 445 } 446 } 447 /* qpd not found in dqm list */ 448 retval = 1; 449 out: 450 mutex_unlock(&dqm->lock); 451 return retval; 452 } 453 454 static int 455 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, 456 unsigned int vmid) 457 { 458 uint32_t pasid_mapping; 459 460 pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 461 ATC_VMID_PASID_MAPPING_VALID; 462 return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, 463 vmid); 464 } 465 466 int init_pipelines(struct device_queue_manager *dqm, 467 unsigned int pipes_num, unsigned int first_pipe) 468 { 469 void *hpdptr; 470 struct mqd_manager *mqd; 471 unsigned int i, err, inx; 472 uint64_t pipe_hpd_addr; 473 474 BUG_ON(!dqm || !dqm->dev); 475 476 pr_debug("kfd: In func %s\n", __func__); 477 478 /* 479 * Allocate memory for the HPDs. This is hardware-owned per-pipe data. 480 * The driver never accesses this memory after zeroing it. 481 * It doesn't even have to be saved/restored on suspend/resume 482 * because it contains no data when there are no active queues. 483 */ 484 485 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num, 486 &dqm->pipeline_mem); 487 488 if (err) { 489 pr_err("kfd: error allocate vidmem num pipes: %d\n", 490 pipes_num); 491 return -ENOMEM; 492 } 493 494 hpdptr = dqm->pipeline_mem->cpu_ptr; 495 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; 496 497 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); 498 499 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 500 if (mqd == NULL) { 501 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 502 return -ENOMEM; 503 } 504 505 for (i = 0; i < pipes_num; i++) { 506 inx = i + first_pipe; 507 /* 508 * HPD buffer on GTT is allocated by amdkfd, no need to waste 509 * space in GTT for pipelines we don't initialize 510 */ 511 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; 512 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); 513 /* = log2(bytes/4)-1 */ 514 kfd2kgd->init_pipeline(dqm->dev->kgd, inx, 515 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); 516 } 517 518 return 0; 519 } 520 521 static int init_scheduler(struct device_queue_manager *dqm) 522 { 523 int retval; 524 525 BUG_ON(!dqm); 526 527 pr_debug("kfd: In %s\n", __func__); 528 529 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm)); 530 return retval; 531 } 532 533 static int initialize_nocpsch(struct device_queue_manager *dqm) 534 { 535 int i; 536 537 BUG_ON(!dqm); 538 539 pr_debug("kfd: In func %s num of pipes: %d\n", 540 __func__, get_pipes_num(dqm)); 541 542 mutex_init(&dqm->lock); 543 INIT_LIST_HEAD(&dqm->queues); 544 dqm->queue_count = dqm->next_pipe_to_allocate = 0; 545 dqm->sdma_queue_count = 0; 546 dqm->allocated_queues = kcalloc(get_pipes_num(dqm), 547 sizeof(unsigned int), GFP_KERNEL); 548 if (!dqm->allocated_queues) { 549 mutex_destroy(&dqm->lock); 550 return -ENOMEM; 551 } 552 553 for (i = 0; i < get_pipes_num(dqm); i++) 554 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; 555 556 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; 557 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1; 558 559 init_scheduler(dqm); 560 return 0; 561 } 562 563 static void uninitialize_nocpsch(struct device_queue_manager *dqm) 564 { 565 int i; 566 567 BUG_ON(!dqm); 568 569 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); 570 571 kfree(dqm->allocated_queues); 572 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 573 kfree(dqm->mqds[i]); 574 mutex_destroy(&dqm->lock); 575 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 576 } 577 578 static int start_nocpsch(struct device_queue_manager *dqm) 579 { 580 return 0; 581 } 582 583 static int stop_nocpsch(struct device_queue_manager *dqm) 584 { 585 return 0; 586 } 587 588 static int allocate_sdma_queue(struct device_queue_manager *dqm, 589 unsigned int *sdma_queue_id) 590 { 591 int bit; 592 593 if (dqm->sdma_bitmap == 0) 594 return -ENOMEM; 595 596 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap, 597 CIK_SDMA_QUEUES); 598 599 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap); 600 *sdma_queue_id = bit; 601 602 return 0; 603 } 604 605 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 606 unsigned int sdma_queue_id) 607 { 608 if (sdma_queue_id >= CIK_SDMA_QUEUES) 609 return; 610 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap); 611 } 612 613 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, 614 struct qcm_process_device *qpd) 615 { 616 uint32_t value = SDMA_ATC; 617 618 if (q->process->is_32bit_user_mode) 619 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); 620 else 621 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( 622 qpd_to_pdd(qpd))); 623 q->properties.sdma_vm_addr = value; 624 } 625 626 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 627 struct queue *q, 628 struct qcm_process_device *qpd) 629 { 630 struct mqd_manager *mqd; 631 int retval; 632 633 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 634 if (!mqd) 635 return -ENOMEM; 636 637 retval = allocate_sdma_queue(dqm, &q->sdma_id); 638 if (retval != 0) 639 return retval; 640 641 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE; 642 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM; 643 644 pr_debug("kfd: sdma id is: %d\n", q->sdma_id); 645 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id); 646 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id); 647 648 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 649 &q->gart_mqd_addr, &q->properties); 650 if (retval != 0) { 651 deallocate_sdma_queue(dqm, q->sdma_id); 652 return retval; 653 } 654 655 init_sdma_vm(dqm, q, qpd); 656 return 0; 657 } 658 659 /* 660 * Device Queue Manager implementation for cp scheduler 661 */ 662 663 static int set_sched_resources(struct device_queue_manager *dqm) 664 { 665 struct scheduling_resources res; 666 unsigned int queue_num, queue_mask; 667 668 BUG_ON(!dqm); 669 670 pr_debug("kfd: In func %s\n", __func__); 671 672 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; 673 queue_mask = (1 << queue_num) - 1; 674 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; 675 res.vmid_mask <<= KFD_VMID_START_OFFSET; 676 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); 677 res.gws_mask = res.oac_mask = res.gds_heap_base = 678 res.gds_heap_size = 0; 679 680 pr_debug("kfd: scheduling resources:\n" 681 " vmid mask: 0x%8X\n" 682 " queue mask: 0x%8llX\n", 683 res.vmid_mask, res.queue_mask); 684 685 return pm_send_set_resources(&dqm->packets, &res); 686 } 687 688 static int initialize_cpsch(struct device_queue_manager *dqm) 689 { 690 int retval; 691 692 BUG_ON(!dqm); 693 694 pr_debug("kfd: In func %s num of pipes: %d\n", 695 __func__, get_pipes_num_cpsch()); 696 697 mutex_init(&dqm->lock); 698 INIT_LIST_HEAD(&dqm->queues); 699 dqm->queue_count = dqm->processes_count = 0; 700 dqm->sdma_queue_count = 0; 701 dqm->active_runlist = false; 702 retval = dqm->ops_asic_specific.initialize(dqm); 703 if (retval != 0) 704 goto fail_init_pipelines; 705 706 return 0; 707 708 fail_init_pipelines: 709 mutex_destroy(&dqm->lock); 710 return retval; 711 } 712 713 static int start_cpsch(struct device_queue_manager *dqm) 714 { 715 struct device_process_node *node; 716 int retval; 717 718 BUG_ON(!dqm); 719 720 retval = 0; 721 722 retval = pm_init(&dqm->packets, dqm); 723 if (retval != 0) 724 goto fail_packet_manager_init; 725 726 retval = set_sched_resources(dqm); 727 if (retval != 0) 728 goto fail_set_sched_resources; 729 730 pr_debug("kfd: allocating fence memory\n"); 731 732 /* allocate fence memory on the gart */ 733 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 734 &dqm->fence_mem); 735 736 if (retval != 0) 737 goto fail_allocate_vidmem; 738 739 dqm->fence_addr = dqm->fence_mem->cpu_ptr; 740 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 741 list_for_each_entry(node, &dqm->queues, list) 742 if (node->qpd->pqm->process && dqm->dev) 743 kfd_bind_process_to_device(dqm->dev, 744 node->qpd->pqm->process); 745 746 execute_queues_cpsch(dqm, true); 747 748 return 0; 749 fail_allocate_vidmem: 750 fail_set_sched_resources: 751 pm_uninit(&dqm->packets); 752 fail_packet_manager_init: 753 return retval; 754 } 755 756 static int stop_cpsch(struct device_queue_manager *dqm) 757 { 758 struct device_process_node *node; 759 struct kfd_process_device *pdd; 760 761 BUG_ON(!dqm); 762 763 destroy_queues_cpsch(dqm, true); 764 765 list_for_each_entry(node, &dqm->queues, list) { 766 pdd = qpd_to_pdd(node->qpd); 767 pdd->bound = false; 768 } 769 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 770 pm_uninit(&dqm->packets); 771 772 return 0; 773 } 774 775 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 776 struct kernel_queue *kq, 777 struct qcm_process_device *qpd) 778 { 779 BUG_ON(!dqm || !kq || !qpd); 780 781 pr_debug("kfd: In func %s\n", __func__); 782 783 mutex_lock(&dqm->lock); 784 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 785 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n", 786 dqm->total_queue_count); 787 mutex_unlock(&dqm->lock); 788 return -EPERM; 789 } 790 791 /* 792 * Unconditionally increment this counter, regardless of the queue's 793 * type or whether the queue is active. 794 */ 795 dqm->total_queue_count++; 796 pr_debug("Total of %d queues are accountable so far\n", 797 dqm->total_queue_count); 798 799 list_add(&kq->list, &qpd->priv_queue_list); 800 dqm->queue_count++; 801 qpd->is_debug = true; 802 execute_queues_cpsch(dqm, false); 803 mutex_unlock(&dqm->lock); 804 805 return 0; 806 } 807 808 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 809 struct kernel_queue *kq, 810 struct qcm_process_device *qpd) 811 { 812 BUG_ON(!dqm || !kq); 813 814 pr_debug("kfd: In %s\n", __func__); 815 816 mutex_lock(&dqm->lock); 817 destroy_queues_cpsch(dqm, false); 818 list_del(&kq->list); 819 dqm->queue_count--; 820 qpd->is_debug = false; 821 execute_queues_cpsch(dqm, false); 822 /* 823 * Unconditionally decrement this counter, regardless of the queue's 824 * type. 825 */ 826 dqm->total_queue_count--; 827 pr_debug("Total of %d queues are accountable so far\n", 828 dqm->total_queue_count); 829 mutex_unlock(&dqm->lock); 830 } 831 832 static void select_sdma_engine_id(struct queue *q) 833 { 834 static int sdma_id; 835 836 q->sdma_id = sdma_id; 837 sdma_id = (sdma_id + 1) % 2; 838 } 839 840 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 841 struct qcm_process_device *qpd, int *allocate_vmid) 842 { 843 int retval; 844 struct mqd_manager *mqd; 845 846 BUG_ON(!dqm || !q || !qpd); 847 848 retval = 0; 849 850 if (allocate_vmid) 851 *allocate_vmid = 0; 852 853 mutex_lock(&dqm->lock); 854 855 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 856 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 857 dqm->total_queue_count); 858 retval = -EPERM; 859 goto out; 860 } 861 862 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 863 select_sdma_engine_id(q); 864 865 mqd = dqm->ops.get_mqd_manager(dqm, 866 get_mqd_type_from_queue_type(q->properties.type)); 867 868 if (mqd == NULL) { 869 mutex_unlock(&dqm->lock); 870 return -ENOMEM; 871 } 872 873 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 874 &q->gart_mqd_addr, &q->properties); 875 if (retval != 0) 876 goto out; 877 878 list_add(&q->list, &qpd->queues_list); 879 if (q->properties.is_active) { 880 dqm->queue_count++; 881 retval = execute_queues_cpsch(dqm, false); 882 } 883 884 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 885 dqm->sdma_queue_count++; 886 /* 887 * Unconditionally increment this counter, regardless of the queue's 888 * type or whether the queue is active. 889 */ 890 dqm->total_queue_count++; 891 892 pr_debug("Total of %d queues are accountable so far\n", 893 dqm->total_queue_count); 894 895 out: 896 mutex_unlock(&dqm->lock); 897 return retval; 898 } 899 900 static int fence_wait_timeout(unsigned int *fence_addr, 901 unsigned int fence_value, 902 unsigned long timeout) 903 { 904 BUG_ON(!fence_addr); 905 timeout += jiffies; 906 907 while (*fence_addr != fence_value) { 908 if (time_after(jiffies, timeout)) { 909 pr_err("kfd: qcm fence wait loop timeout expired\n"); 910 return -ETIME; 911 } 912 schedule(); 913 } 914 915 return 0; 916 } 917 918 static int destroy_sdma_queues(struct device_queue_manager *dqm, 919 unsigned int sdma_engine) 920 { 921 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, 922 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 923 sdma_engine); 924 } 925 926 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) 927 { 928 int retval; 929 930 BUG_ON(!dqm); 931 932 retval = 0; 933 934 if (lock) 935 mutex_lock(&dqm->lock); 936 if (dqm->active_runlist == false) 937 goto out; 938 939 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n", 940 dqm->sdma_queue_count); 941 942 if (dqm->sdma_queue_count > 0) { 943 destroy_sdma_queues(dqm, 0); 944 destroy_sdma_queues(dqm, 1); 945 } 946 947 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, 948 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); 949 if (retval != 0) 950 goto out; 951 952 *dqm->fence_addr = KFD_FENCE_INIT; 953 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, 954 KFD_FENCE_COMPLETED); 955 /* should be timed out */ 956 fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, 957 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); 958 pm_release_ib(&dqm->packets); 959 dqm->active_runlist = false; 960 961 out: 962 if (lock) 963 mutex_unlock(&dqm->lock); 964 return retval; 965 } 966 967 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) 968 { 969 int retval; 970 971 BUG_ON(!dqm); 972 973 if (lock) 974 mutex_lock(&dqm->lock); 975 976 retval = destroy_queues_cpsch(dqm, false); 977 if (retval != 0) { 978 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); 979 goto out; 980 } 981 982 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { 983 retval = 0; 984 goto out; 985 } 986 987 if (dqm->active_runlist) { 988 retval = 0; 989 goto out; 990 } 991 992 retval = pm_send_runlist(&dqm->packets, &dqm->queues); 993 if (retval != 0) { 994 pr_err("kfd: failed to execute runlist"); 995 goto out; 996 } 997 dqm->active_runlist = true; 998 999 out: 1000 if (lock) 1001 mutex_unlock(&dqm->lock); 1002 return retval; 1003 } 1004 1005 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 1006 struct qcm_process_device *qpd, 1007 struct queue *q) 1008 { 1009 int retval; 1010 struct mqd_manager *mqd; 1011 1012 BUG_ON(!dqm || !qpd || !q); 1013 1014 retval = 0; 1015 1016 /* remove queue from list to prevent rescheduling after preemption */ 1017 mutex_lock(&dqm->lock); 1018 mqd = dqm->ops.get_mqd_manager(dqm, 1019 get_mqd_type_from_queue_type(q->properties.type)); 1020 if (!mqd) { 1021 retval = -ENOMEM; 1022 goto failed; 1023 } 1024 1025 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1026 dqm->sdma_queue_count--; 1027 1028 list_del(&q->list); 1029 if (q->properties.is_active) 1030 dqm->queue_count--; 1031 1032 execute_queues_cpsch(dqm, false); 1033 1034 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 1035 1036 /* 1037 * Unconditionally decrement this counter, regardless of the queue's 1038 * type 1039 */ 1040 dqm->total_queue_count--; 1041 pr_debug("Total of %d queues are accountable so far\n", 1042 dqm->total_queue_count); 1043 1044 mutex_unlock(&dqm->lock); 1045 1046 return 0; 1047 1048 failed: 1049 mutex_unlock(&dqm->lock); 1050 return retval; 1051 } 1052 1053 /* 1054 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 1055 * stay in user mode. 1056 */ 1057 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 1058 /* APE1 limit is inclusive and 64K aligned. */ 1059 #define APE1_LIMIT_ALIGNMENT 0xFFFF 1060 1061 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 1062 struct qcm_process_device *qpd, 1063 enum cache_policy default_policy, 1064 enum cache_policy alternate_policy, 1065 void __user *alternate_aperture_base, 1066 uint64_t alternate_aperture_size) 1067 { 1068 bool retval; 1069 1070 pr_debug("kfd: In func %s\n", __func__); 1071 1072 mutex_lock(&dqm->lock); 1073 1074 if (alternate_aperture_size == 0) { 1075 /* base > limit disables APE1 */ 1076 qpd->sh_mem_ape1_base = 1; 1077 qpd->sh_mem_ape1_limit = 0; 1078 } else { 1079 /* 1080 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 1081 * SH_MEM_APE1_BASE[31:0], 0x0000 } 1082 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 1083 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 1084 * Verify that the base and size parameters can be 1085 * represented in this format and convert them. 1086 * Additionally restrict APE1 to user-mode addresses. 1087 */ 1088 1089 uint64_t base = (uintptr_t)alternate_aperture_base; 1090 uint64_t limit = base + alternate_aperture_size - 1; 1091 1092 if (limit <= base) 1093 goto out; 1094 1095 if ((base & APE1_FIXED_BITS_MASK) != 0) 1096 goto out; 1097 1098 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) 1099 goto out; 1100 1101 qpd->sh_mem_ape1_base = base >> 16; 1102 qpd->sh_mem_ape1_limit = limit >> 16; 1103 } 1104 1105 retval = dqm->ops_asic_specific.set_cache_memory_policy( 1106 dqm, 1107 qpd, 1108 default_policy, 1109 alternate_policy, 1110 alternate_aperture_base, 1111 alternate_aperture_size); 1112 1113 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 1114 program_sh_mem_settings(dqm, qpd); 1115 1116 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 1117 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 1118 qpd->sh_mem_ape1_limit); 1119 1120 mutex_unlock(&dqm->lock); 1121 return retval; 1122 1123 out: 1124 mutex_unlock(&dqm->lock); 1125 return false; 1126 } 1127 1128 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) 1129 { 1130 struct device_queue_manager *dqm; 1131 1132 BUG_ON(!dev); 1133 1134 pr_debug("kfd: loading device queue manager\n"); 1135 1136 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); 1137 if (!dqm) 1138 return NULL; 1139 1140 dqm->dev = dev; 1141 switch (sched_policy) { 1142 case KFD_SCHED_POLICY_HWS: 1143 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 1144 /* initialize dqm for cp scheduling */ 1145 dqm->ops.create_queue = create_queue_cpsch; 1146 dqm->ops.initialize = initialize_cpsch; 1147 dqm->ops.start = start_cpsch; 1148 dqm->ops.stop = stop_cpsch; 1149 dqm->ops.destroy_queue = destroy_queue_cpsch; 1150 dqm->ops.update_queue = update_queue; 1151 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1152 dqm->ops.register_process = register_process_nocpsch; 1153 dqm->ops.unregister_process = unregister_process_nocpsch; 1154 dqm->ops.uninitialize = uninitialize_nocpsch; 1155 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 1156 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 1157 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1158 break; 1159 case KFD_SCHED_POLICY_NO_HWS: 1160 /* initialize dqm for no cp scheduling */ 1161 dqm->ops.start = start_nocpsch; 1162 dqm->ops.stop = stop_nocpsch; 1163 dqm->ops.create_queue = create_queue_nocpsch; 1164 dqm->ops.destroy_queue = destroy_queue_nocpsch; 1165 dqm->ops.update_queue = update_queue; 1166 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1167 dqm->ops.register_process = register_process_nocpsch; 1168 dqm->ops.unregister_process = unregister_process_nocpsch; 1169 dqm->ops.initialize = initialize_nocpsch; 1170 dqm->ops.uninitialize = uninitialize_nocpsch; 1171 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1172 break; 1173 default: 1174 BUG(); 1175 break; 1176 } 1177 1178 switch (dev->device_info->asic_family) { 1179 case CHIP_CARRIZO: 1180 device_queue_manager_init_vi(&dqm->ops_asic_specific); 1181 break; 1182 1183 case CHIP_KAVERI: 1184 device_queue_manager_init_cik(&dqm->ops_asic_specific); 1185 break; 1186 } 1187 1188 if (dqm->ops.initialize(dqm) != 0) { 1189 kfree(dqm); 1190 return NULL; 1191 } 1192 1193 return dqm; 1194 } 1195 1196 void device_queue_manager_uninit(struct device_queue_manager *dqm) 1197 { 1198 BUG_ON(!dqm); 1199 1200 dqm->ops.uninitialize(dqm); 1201 kfree(dqm); 1202 } 1203