1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/slab.h> 25 #include <linux/list.h> 26 #include <linux/types.h> 27 #include <linux/printk.h> 28 #include <linux/bitops.h> 29 #include <linux/sched.h> 30 #include "kfd_priv.h" 31 #include "kfd_device_queue_manager.h" 32 #include "kfd_mqd_manager.h" 33 #include "cik_regs.h" 34 #include "kfd_kernel_queue.h" 35 36 /* Size of the per-pipe EOP queue */ 37 #define CIK_HPD_EOP_BYTES_LOG2 11 38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 39 40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 41 unsigned int pasid, unsigned int vmid); 42 43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 44 struct queue *q, 45 struct qcm_process_device *qpd); 46 47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); 48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); 49 50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 51 struct queue *q, 52 struct qcm_process_device *qpd); 53 54 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 55 unsigned int sdma_queue_id); 56 57 static inline 58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 59 { 60 if (type == KFD_QUEUE_TYPE_SDMA) 61 return KFD_MQD_TYPE_SDMA; 62 return KFD_MQD_TYPE_CP; 63 } 64 65 static inline unsigned int get_first_pipe(struct device_queue_manager *dqm) 66 { 67 BUG_ON(!dqm); 68 return dqm->dev->shared_resources.first_compute_pipe; 69 } 70 71 static inline unsigned int get_pipes_num_cpsch(void) 72 { 73 return PIPE_PER_ME_CP_SCHEDULING; 74 } 75 76 void program_sh_mem_settings(struct device_queue_manager *dqm, 77 struct qcm_process_device *qpd) 78 { 79 return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, 80 qpd->sh_mem_config, 81 qpd->sh_mem_ape1_base, 82 qpd->sh_mem_ape1_limit, 83 qpd->sh_mem_bases); 84 } 85 86 static int allocate_vmid(struct device_queue_manager *dqm, 87 struct qcm_process_device *qpd, 88 struct queue *q) 89 { 90 int bit, allocated_vmid; 91 92 if (dqm->vmid_bitmap == 0) 93 return -ENOMEM; 94 95 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); 96 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 97 98 /* Kaveri kfd vmid's starts from vmid 8 */ 99 allocated_vmid = bit + KFD_VMID_START_OFFSET; 100 pr_debug("kfd: vmid allocation %d\n", allocated_vmid); 101 qpd->vmid = allocated_vmid; 102 q->properties.vmid = allocated_vmid; 103 104 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); 105 program_sh_mem_settings(dqm, qpd); 106 107 return 0; 108 } 109 110 static void deallocate_vmid(struct device_queue_manager *dqm, 111 struct qcm_process_device *qpd, 112 struct queue *q) 113 { 114 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 115 116 /* Release the vmid mapping */ 117 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 118 119 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 120 qpd->vmid = 0; 121 q->properties.vmid = 0; 122 } 123 124 static int create_queue_nocpsch(struct device_queue_manager *dqm, 125 struct queue *q, 126 struct qcm_process_device *qpd, 127 int *allocated_vmid) 128 { 129 int retval; 130 131 BUG_ON(!dqm || !q || !qpd || !allocated_vmid); 132 133 pr_debug("kfd: In func %s\n", __func__); 134 print_queue(q); 135 136 mutex_lock(&dqm->lock); 137 138 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 139 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 140 dqm->total_queue_count); 141 mutex_unlock(&dqm->lock); 142 return -EPERM; 143 } 144 145 if (list_empty(&qpd->queues_list)) { 146 retval = allocate_vmid(dqm, qpd, q); 147 if (retval != 0) { 148 mutex_unlock(&dqm->lock); 149 return retval; 150 } 151 } 152 *allocated_vmid = qpd->vmid; 153 q->properties.vmid = qpd->vmid; 154 155 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 156 retval = create_compute_queue_nocpsch(dqm, q, qpd); 157 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 158 retval = create_sdma_queue_nocpsch(dqm, q, qpd); 159 160 if (retval != 0) { 161 if (list_empty(&qpd->queues_list)) { 162 deallocate_vmid(dqm, qpd, q); 163 *allocated_vmid = 0; 164 } 165 mutex_unlock(&dqm->lock); 166 return retval; 167 } 168 169 list_add(&q->list, &qpd->queues_list); 170 if (q->properties.is_active) 171 dqm->queue_count++; 172 173 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 174 dqm->sdma_queue_count++; 175 176 /* 177 * Unconditionally increment this counter, regardless of the queue's 178 * type or whether the queue is active. 179 */ 180 dqm->total_queue_count++; 181 pr_debug("Total of %d queues are accountable so far\n", 182 dqm->total_queue_count); 183 184 mutex_unlock(&dqm->lock); 185 return 0; 186 } 187 188 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 189 { 190 bool set; 191 int pipe, bit, i; 192 193 set = false; 194 195 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm); 196 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) { 197 if (dqm->allocated_queues[pipe] != 0) { 198 bit = find_first_bit( 199 (unsigned long *)&dqm->allocated_queues[pipe], 200 QUEUES_PER_PIPE); 201 202 clear_bit(bit, 203 (unsigned long *)&dqm->allocated_queues[pipe]); 204 q->pipe = pipe; 205 q->queue = bit; 206 set = true; 207 break; 208 } 209 } 210 211 if (set == false) 212 return -EBUSY; 213 214 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", 215 __func__, q->pipe, q->queue); 216 /* horizontal hqd allocation */ 217 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); 218 219 return 0; 220 } 221 222 static inline void deallocate_hqd(struct device_queue_manager *dqm, 223 struct queue *q) 224 { 225 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); 226 } 227 228 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 229 struct queue *q, 230 struct qcm_process_device *qpd) 231 { 232 int retval; 233 struct mqd_manager *mqd; 234 235 BUG_ON(!dqm || !q || !qpd); 236 237 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 238 if (mqd == NULL) 239 return -ENOMEM; 240 241 retval = allocate_hqd(dqm, q); 242 if (retval != 0) 243 return retval; 244 245 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 246 &q->gart_mqd_addr, &q->properties); 247 if (retval != 0) { 248 deallocate_hqd(dqm, q); 249 return retval; 250 } 251 252 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", 253 q->pipe, 254 q->queue); 255 256 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 257 q->queue, (uint32_t __user *) q->properties.write_ptr); 258 if (retval != 0) { 259 deallocate_hqd(dqm, q); 260 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 261 return retval; 262 } 263 264 return 0; 265 } 266 267 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 268 struct qcm_process_device *qpd, 269 struct queue *q) 270 { 271 int retval; 272 struct mqd_manager *mqd; 273 274 BUG_ON(!dqm || !q || !q->mqd || !qpd); 275 276 retval = 0; 277 278 pr_debug("kfd: In Func %s\n", __func__); 279 280 mutex_lock(&dqm->lock); 281 282 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 283 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 284 if (mqd == NULL) { 285 retval = -ENOMEM; 286 goto out; 287 } 288 deallocate_hqd(dqm, q); 289 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 290 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 291 if (mqd == NULL) { 292 retval = -ENOMEM; 293 goto out; 294 } 295 dqm->sdma_queue_count--; 296 deallocate_sdma_queue(dqm, q->sdma_id); 297 } else { 298 pr_debug("q->properties.type is invalid (%d)\n", 299 q->properties.type); 300 retval = -EINVAL; 301 goto out; 302 } 303 304 retval = mqd->destroy_mqd(mqd, q->mqd, 305 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 306 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, 307 q->pipe, q->queue); 308 309 if (retval != 0) 310 goto out; 311 312 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 313 314 list_del(&q->list); 315 if (list_empty(&qpd->queues_list)) 316 deallocate_vmid(dqm, qpd, q); 317 if (q->properties.is_active) 318 dqm->queue_count--; 319 320 /* 321 * Unconditionally decrement this counter, regardless of the queue's 322 * type 323 */ 324 dqm->total_queue_count--; 325 pr_debug("Total of %d queues are accountable so far\n", 326 dqm->total_queue_count); 327 328 out: 329 mutex_unlock(&dqm->lock); 330 return retval; 331 } 332 333 static int update_queue(struct device_queue_manager *dqm, struct queue *q) 334 { 335 int retval; 336 struct mqd_manager *mqd; 337 bool prev_active = false; 338 339 BUG_ON(!dqm || !q || !q->mqd); 340 341 mutex_lock(&dqm->lock); 342 mqd = dqm->ops.get_mqd_manager(dqm, 343 get_mqd_type_from_queue_type(q->properties.type)); 344 if (mqd == NULL) { 345 mutex_unlock(&dqm->lock); 346 return -ENOMEM; 347 } 348 349 if (q->properties.is_active == true) 350 prev_active = true; 351 352 /* 353 * 354 * check active state vs. the previous state 355 * and modify counter accordingly 356 */ 357 retval = mqd->update_mqd(mqd, q->mqd, &q->properties); 358 if ((q->properties.is_active == true) && (prev_active == false)) 359 dqm->queue_count++; 360 else if ((q->properties.is_active == false) && (prev_active == true)) 361 dqm->queue_count--; 362 363 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 364 retval = execute_queues_cpsch(dqm, false); 365 366 mutex_unlock(&dqm->lock); 367 return retval; 368 } 369 370 static struct mqd_manager *get_mqd_manager_nocpsch( 371 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) 372 { 373 struct mqd_manager *mqd; 374 375 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); 376 377 pr_debug("kfd: In func %s mqd type %d\n", __func__, type); 378 379 mqd = dqm->mqds[type]; 380 if (!mqd) { 381 mqd = mqd_manager_init(type, dqm->dev); 382 if (mqd == NULL) 383 pr_err("kfd: mqd manager is NULL"); 384 dqm->mqds[type] = mqd; 385 } 386 387 return mqd; 388 } 389 390 static int register_process_nocpsch(struct device_queue_manager *dqm, 391 struct qcm_process_device *qpd) 392 { 393 struct device_process_node *n; 394 int retval; 395 396 BUG_ON(!dqm || !qpd); 397 398 pr_debug("kfd: In func %s\n", __func__); 399 400 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); 401 if (!n) 402 return -ENOMEM; 403 404 n->qpd = qpd; 405 406 mutex_lock(&dqm->lock); 407 list_add(&n->list, &dqm->queues); 408 409 retval = dqm->ops_asic_specific.register_process(dqm, qpd); 410 411 dqm->processes_count++; 412 413 mutex_unlock(&dqm->lock); 414 415 return retval; 416 } 417 418 static int unregister_process_nocpsch(struct device_queue_manager *dqm, 419 struct qcm_process_device *qpd) 420 { 421 int retval; 422 struct device_process_node *cur, *next; 423 424 BUG_ON(!dqm || !qpd); 425 426 BUG_ON(!list_empty(&qpd->queues_list)); 427 428 pr_debug("kfd: In func %s\n", __func__); 429 430 retval = 0; 431 mutex_lock(&dqm->lock); 432 433 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 434 if (qpd == cur->qpd) { 435 list_del(&cur->list); 436 kfree(cur); 437 dqm->processes_count--; 438 goto out; 439 } 440 } 441 /* qpd not found in dqm list */ 442 retval = 1; 443 out: 444 mutex_unlock(&dqm->lock); 445 return retval; 446 } 447 448 static int 449 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, 450 unsigned int vmid) 451 { 452 uint32_t pasid_mapping; 453 454 pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 455 ATC_VMID_PASID_MAPPING_VALID; 456 return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, 457 vmid); 458 } 459 460 int init_pipelines(struct device_queue_manager *dqm, 461 unsigned int pipes_num, unsigned int first_pipe) 462 { 463 void *hpdptr; 464 struct mqd_manager *mqd; 465 unsigned int i, err, inx; 466 uint64_t pipe_hpd_addr; 467 468 BUG_ON(!dqm || !dqm->dev); 469 470 pr_debug("kfd: In func %s\n", __func__); 471 472 /* 473 * Allocate memory for the HPDs. This is hardware-owned per-pipe data. 474 * The driver never accesses this memory after zeroing it. 475 * It doesn't even have to be saved/restored on suspend/resume 476 * because it contains no data when there are no active queues. 477 */ 478 479 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num, 480 &dqm->pipeline_mem); 481 482 if (err) { 483 pr_err("kfd: error allocate vidmem num pipes: %d\n", 484 pipes_num); 485 return -ENOMEM; 486 } 487 488 hpdptr = dqm->pipeline_mem->cpu_ptr; 489 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; 490 491 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); 492 493 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 494 if (mqd == NULL) { 495 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 496 return -ENOMEM; 497 } 498 499 for (i = 0; i < pipes_num; i++) { 500 inx = i + first_pipe; 501 /* 502 * HPD buffer on GTT is allocated by amdkfd, no need to waste 503 * space in GTT for pipelines we don't initialize 504 */ 505 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; 506 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); 507 /* = log2(bytes/4)-1 */ 508 kfd2kgd->init_pipeline(dqm->dev->kgd, inx, 509 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); 510 } 511 512 return 0; 513 } 514 515 static int init_scheduler(struct device_queue_manager *dqm) 516 { 517 int retval; 518 519 BUG_ON(!dqm); 520 521 pr_debug("kfd: In %s\n", __func__); 522 523 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm)); 524 return retval; 525 } 526 527 static int initialize_nocpsch(struct device_queue_manager *dqm) 528 { 529 int i; 530 531 BUG_ON(!dqm); 532 533 pr_debug("kfd: In func %s num of pipes: %d\n", 534 __func__, get_pipes_num(dqm)); 535 536 mutex_init(&dqm->lock); 537 INIT_LIST_HEAD(&dqm->queues); 538 dqm->queue_count = dqm->next_pipe_to_allocate = 0; 539 dqm->sdma_queue_count = 0; 540 dqm->allocated_queues = kcalloc(get_pipes_num(dqm), 541 sizeof(unsigned int), GFP_KERNEL); 542 if (!dqm->allocated_queues) { 543 mutex_destroy(&dqm->lock); 544 return -ENOMEM; 545 } 546 547 for (i = 0; i < get_pipes_num(dqm); i++) 548 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; 549 550 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; 551 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1; 552 553 init_scheduler(dqm); 554 return 0; 555 } 556 557 static void uninitialize_nocpsch(struct device_queue_manager *dqm) 558 { 559 int i; 560 561 BUG_ON(!dqm); 562 563 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); 564 565 kfree(dqm->allocated_queues); 566 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 567 kfree(dqm->mqds[i]); 568 mutex_destroy(&dqm->lock); 569 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 570 } 571 572 static int start_nocpsch(struct device_queue_manager *dqm) 573 { 574 return 0; 575 } 576 577 static int stop_nocpsch(struct device_queue_manager *dqm) 578 { 579 return 0; 580 } 581 582 static int allocate_sdma_queue(struct device_queue_manager *dqm, 583 unsigned int *sdma_queue_id) 584 { 585 int bit; 586 587 if (dqm->sdma_bitmap == 0) 588 return -ENOMEM; 589 590 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap, 591 CIK_SDMA_QUEUES); 592 593 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap); 594 *sdma_queue_id = bit; 595 596 return 0; 597 } 598 599 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 600 unsigned int sdma_queue_id) 601 { 602 if (sdma_queue_id >= CIK_SDMA_QUEUES) 603 return; 604 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap); 605 } 606 607 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, 608 struct qcm_process_device *qpd) 609 { 610 uint32_t value = SDMA_ATC; 611 612 if (q->process->is_32bit_user_mode) 613 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); 614 else 615 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( 616 qpd_to_pdd(qpd))); 617 q->properties.sdma_vm_addr = value; 618 } 619 620 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 621 struct queue *q, 622 struct qcm_process_device *qpd) 623 { 624 struct mqd_manager *mqd; 625 int retval; 626 627 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 628 if (!mqd) 629 return -ENOMEM; 630 631 retval = allocate_sdma_queue(dqm, &q->sdma_id); 632 if (retval != 0) 633 return retval; 634 635 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE; 636 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM; 637 638 pr_debug("kfd: sdma id is: %d\n", q->sdma_id); 639 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id); 640 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id); 641 642 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 643 &q->gart_mqd_addr, &q->properties); 644 if (retval != 0) { 645 deallocate_sdma_queue(dqm, q->sdma_id); 646 return retval; 647 } 648 649 init_sdma_vm(dqm, q, qpd); 650 return 0; 651 } 652 653 /* 654 * Device Queue Manager implementation for cp scheduler 655 */ 656 657 static int set_sched_resources(struct device_queue_manager *dqm) 658 { 659 struct scheduling_resources res; 660 unsigned int queue_num, queue_mask; 661 662 BUG_ON(!dqm); 663 664 pr_debug("kfd: In func %s\n", __func__); 665 666 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; 667 queue_mask = (1 << queue_num) - 1; 668 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; 669 res.vmid_mask <<= KFD_VMID_START_OFFSET; 670 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); 671 res.gws_mask = res.oac_mask = res.gds_heap_base = 672 res.gds_heap_size = 0; 673 674 pr_debug("kfd: scheduling resources:\n" 675 " vmid mask: 0x%8X\n" 676 " queue mask: 0x%8llX\n", 677 res.vmid_mask, res.queue_mask); 678 679 return pm_send_set_resources(&dqm->packets, &res); 680 } 681 682 static int initialize_cpsch(struct device_queue_manager *dqm) 683 { 684 int retval; 685 686 BUG_ON(!dqm); 687 688 pr_debug("kfd: In func %s num of pipes: %d\n", 689 __func__, get_pipes_num_cpsch()); 690 691 mutex_init(&dqm->lock); 692 INIT_LIST_HEAD(&dqm->queues); 693 dqm->queue_count = dqm->processes_count = 0; 694 dqm->sdma_queue_count = 0; 695 dqm->active_runlist = false; 696 retval = dqm->ops_asic_specific.initialize(dqm); 697 if (retval != 0) 698 goto fail_init_pipelines; 699 700 return 0; 701 702 fail_init_pipelines: 703 mutex_destroy(&dqm->lock); 704 return retval; 705 } 706 707 static int start_cpsch(struct device_queue_manager *dqm) 708 { 709 struct device_process_node *node; 710 int retval; 711 712 BUG_ON(!dqm); 713 714 retval = 0; 715 716 retval = pm_init(&dqm->packets, dqm); 717 if (retval != 0) 718 goto fail_packet_manager_init; 719 720 retval = set_sched_resources(dqm); 721 if (retval != 0) 722 goto fail_set_sched_resources; 723 724 pr_debug("kfd: allocating fence memory\n"); 725 726 /* allocate fence memory on the gart */ 727 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 728 &dqm->fence_mem); 729 730 if (retval != 0) 731 goto fail_allocate_vidmem; 732 733 dqm->fence_addr = dqm->fence_mem->cpu_ptr; 734 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 735 list_for_each_entry(node, &dqm->queues, list) 736 if (node->qpd->pqm->process && dqm->dev) 737 kfd_bind_process_to_device(dqm->dev, 738 node->qpd->pqm->process); 739 740 execute_queues_cpsch(dqm, true); 741 742 return 0; 743 fail_allocate_vidmem: 744 fail_set_sched_resources: 745 pm_uninit(&dqm->packets); 746 fail_packet_manager_init: 747 return retval; 748 } 749 750 static int stop_cpsch(struct device_queue_manager *dqm) 751 { 752 struct device_process_node *node; 753 struct kfd_process_device *pdd; 754 755 BUG_ON(!dqm); 756 757 destroy_queues_cpsch(dqm, true); 758 759 list_for_each_entry(node, &dqm->queues, list) { 760 pdd = qpd_to_pdd(node->qpd); 761 pdd->bound = false; 762 } 763 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 764 pm_uninit(&dqm->packets); 765 766 return 0; 767 } 768 769 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 770 struct kernel_queue *kq, 771 struct qcm_process_device *qpd) 772 { 773 BUG_ON(!dqm || !kq || !qpd); 774 775 pr_debug("kfd: In func %s\n", __func__); 776 777 mutex_lock(&dqm->lock); 778 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 779 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n", 780 dqm->total_queue_count); 781 mutex_unlock(&dqm->lock); 782 return -EPERM; 783 } 784 785 /* 786 * Unconditionally increment this counter, regardless of the queue's 787 * type or whether the queue is active. 788 */ 789 dqm->total_queue_count++; 790 pr_debug("Total of %d queues are accountable so far\n", 791 dqm->total_queue_count); 792 793 list_add(&kq->list, &qpd->priv_queue_list); 794 dqm->queue_count++; 795 qpd->is_debug = true; 796 execute_queues_cpsch(dqm, false); 797 mutex_unlock(&dqm->lock); 798 799 return 0; 800 } 801 802 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 803 struct kernel_queue *kq, 804 struct qcm_process_device *qpd) 805 { 806 BUG_ON(!dqm || !kq); 807 808 pr_debug("kfd: In %s\n", __func__); 809 810 mutex_lock(&dqm->lock); 811 destroy_queues_cpsch(dqm, false); 812 list_del(&kq->list); 813 dqm->queue_count--; 814 qpd->is_debug = false; 815 execute_queues_cpsch(dqm, false); 816 /* 817 * Unconditionally decrement this counter, regardless of the queue's 818 * type. 819 */ 820 dqm->total_queue_count--; 821 pr_debug("Total of %d queues are accountable so far\n", 822 dqm->total_queue_count); 823 mutex_unlock(&dqm->lock); 824 } 825 826 static void select_sdma_engine_id(struct queue *q) 827 { 828 static int sdma_id; 829 830 q->sdma_id = sdma_id; 831 sdma_id = (sdma_id + 1) % 2; 832 } 833 834 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 835 struct qcm_process_device *qpd, int *allocate_vmid) 836 { 837 int retval; 838 struct mqd_manager *mqd; 839 840 BUG_ON(!dqm || !q || !qpd); 841 842 retval = 0; 843 844 if (allocate_vmid) 845 *allocate_vmid = 0; 846 847 mutex_lock(&dqm->lock); 848 849 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 850 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 851 dqm->total_queue_count); 852 retval = -EPERM; 853 goto out; 854 } 855 856 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 857 select_sdma_engine_id(q); 858 859 mqd = dqm->ops.get_mqd_manager(dqm, 860 get_mqd_type_from_queue_type(q->properties.type)); 861 862 if (mqd == NULL) { 863 mutex_unlock(&dqm->lock); 864 return -ENOMEM; 865 } 866 867 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 868 &q->gart_mqd_addr, &q->properties); 869 if (retval != 0) 870 goto out; 871 872 list_add(&q->list, &qpd->queues_list); 873 if (q->properties.is_active) { 874 dqm->queue_count++; 875 retval = execute_queues_cpsch(dqm, false); 876 } 877 878 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 879 dqm->sdma_queue_count++; 880 /* 881 * Unconditionally increment this counter, regardless of the queue's 882 * type or whether the queue is active. 883 */ 884 dqm->total_queue_count++; 885 886 pr_debug("Total of %d queues are accountable so far\n", 887 dqm->total_queue_count); 888 889 out: 890 mutex_unlock(&dqm->lock); 891 return retval; 892 } 893 894 static int fence_wait_timeout(unsigned int *fence_addr, 895 unsigned int fence_value, 896 unsigned long timeout) 897 { 898 BUG_ON(!fence_addr); 899 timeout += jiffies; 900 901 while (*fence_addr != fence_value) { 902 if (time_after(jiffies, timeout)) { 903 pr_err("kfd: qcm fence wait loop timeout expired\n"); 904 return -ETIME; 905 } 906 schedule(); 907 } 908 909 return 0; 910 } 911 912 static int destroy_sdma_queues(struct device_queue_manager *dqm, 913 unsigned int sdma_engine) 914 { 915 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, 916 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 917 sdma_engine); 918 } 919 920 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) 921 { 922 int retval; 923 924 BUG_ON(!dqm); 925 926 retval = 0; 927 928 if (lock) 929 mutex_lock(&dqm->lock); 930 if (dqm->active_runlist == false) 931 goto out; 932 933 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n", 934 dqm->sdma_queue_count); 935 936 if (dqm->sdma_queue_count > 0) { 937 destroy_sdma_queues(dqm, 0); 938 destroy_sdma_queues(dqm, 1); 939 } 940 941 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, 942 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); 943 if (retval != 0) 944 goto out; 945 946 *dqm->fence_addr = KFD_FENCE_INIT; 947 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, 948 KFD_FENCE_COMPLETED); 949 /* should be timed out */ 950 fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, 951 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); 952 pm_release_ib(&dqm->packets); 953 dqm->active_runlist = false; 954 955 out: 956 if (lock) 957 mutex_unlock(&dqm->lock); 958 return retval; 959 } 960 961 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) 962 { 963 int retval; 964 965 BUG_ON(!dqm); 966 967 if (lock) 968 mutex_lock(&dqm->lock); 969 970 retval = destroy_queues_cpsch(dqm, false); 971 if (retval != 0) { 972 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); 973 goto out; 974 } 975 976 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { 977 retval = 0; 978 goto out; 979 } 980 981 if (dqm->active_runlist) { 982 retval = 0; 983 goto out; 984 } 985 986 retval = pm_send_runlist(&dqm->packets, &dqm->queues); 987 if (retval != 0) { 988 pr_err("kfd: failed to execute runlist"); 989 goto out; 990 } 991 dqm->active_runlist = true; 992 993 out: 994 if (lock) 995 mutex_unlock(&dqm->lock); 996 return retval; 997 } 998 999 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 1000 struct qcm_process_device *qpd, 1001 struct queue *q) 1002 { 1003 int retval; 1004 struct mqd_manager *mqd; 1005 1006 BUG_ON(!dqm || !qpd || !q); 1007 1008 retval = 0; 1009 1010 /* remove queue from list to prevent rescheduling after preemption */ 1011 mutex_lock(&dqm->lock); 1012 mqd = dqm->ops.get_mqd_manager(dqm, 1013 get_mqd_type_from_queue_type(q->properties.type)); 1014 if (!mqd) { 1015 retval = -ENOMEM; 1016 goto failed; 1017 } 1018 1019 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1020 dqm->sdma_queue_count--; 1021 1022 list_del(&q->list); 1023 if (q->properties.is_active) 1024 dqm->queue_count--; 1025 1026 execute_queues_cpsch(dqm, false); 1027 1028 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 1029 1030 /* 1031 * Unconditionally decrement this counter, regardless of the queue's 1032 * type 1033 */ 1034 dqm->total_queue_count--; 1035 pr_debug("Total of %d queues are accountable so far\n", 1036 dqm->total_queue_count); 1037 1038 mutex_unlock(&dqm->lock); 1039 1040 return 0; 1041 1042 failed: 1043 mutex_unlock(&dqm->lock); 1044 return retval; 1045 } 1046 1047 /* 1048 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 1049 * stay in user mode. 1050 */ 1051 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 1052 /* APE1 limit is inclusive and 64K aligned. */ 1053 #define APE1_LIMIT_ALIGNMENT 0xFFFF 1054 1055 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 1056 struct qcm_process_device *qpd, 1057 enum cache_policy default_policy, 1058 enum cache_policy alternate_policy, 1059 void __user *alternate_aperture_base, 1060 uint64_t alternate_aperture_size) 1061 { 1062 bool retval; 1063 1064 pr_debug("kfd: In func %s\n", __func__); 1065 1066 mutex_lock(&dqm->lock); 1067 1068 if (alternate_aperture_size == 0) { 1069 /* base > limit disables APE1 */ 1070 qpd->sh_mem_ape1_base = 1; 1071 qpd->sh_mem_ape1_limit = 0; 1072 } else { 1073 /* 1074 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 1075 * SH_MEM_APE1_BASE[31:0], 0x0000 } 1076 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 1077 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 1078 * Verify that the base and size parameters can be 1079 * represented in this format and convert them. 1080 * Additionally restrict APE1 to user-mode addresses. 1081 */ 1082 1083 uint64_t base = (uintptr_t)alternate_aperture_base; 1084 uint64_t limit = base + alternate_aperture_size - 1; 1085 1086 if (limit <= base) 1087 goto out; 1088 1089 if ((base & APE1_FIXED_BITS_MASK) != 0) 1090 goto out; 1091 1092 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) 1093 goto out; 1094 1095 qpd->sh_mem_ape1_base = base >> 16; 1096 qpd->sh_mem_ape1_limit = limit >> 16; 1097 } 1098 1099 retval = dqm->ops_asic_specific.set_cache_memory_policy( 1100 dqm, 1101 qpd, 1102 default_policy, 1103 alternate_policy, 1104 alternate_aperture_base, 1105 alternate_aperture_size); 1106 1107 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 1108 program_sh_mem_settings(dqm, qpd); 1109 1110 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 1111 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 1112 qpd->sh_mem_ape1_limit); 1113 1114 mutex_unlock(&dqm->lock); 1115 return retval; 1116 1117 out: 1118 mutex_unlock(&dqm->lock); 1119 return false; 1120 } 1121 1122 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) 1123 { 1124 struct device_queue_manager *dqm; 1125 1126 BUG_ON(!dev); 1127 1128 pr_debug("kfd: loading device queue manager\n"); 1129 1130 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); 1131 if (!dqm) 1132 return NULL; 1133 1134 dqm->dev = dev; 1135 switch (sched_policy) { 1136 case KFD_SCHED_POLICY_HWS: 1137 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 1138 /* initialize dqm for cp scheduling */ 1139 dqm->ops.create_queue = create_queue_cpsch; 1140 dqm->ops.initialize = initialize_cpsch; 1141 dqm->ops.start = start_cpsch; 1142 dqm->ops.stop = stop_cpsch; 1143 dqm->ops.destroy_queue = destroy_queue_cpsch; 1144 dqm->ops.update_queue = update_queue; 1145 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1146 dqm->ops.register_process = register_process_nocpsch; 1147 dqm->ops.unregister_process = unregister_process_nocpsch; 1148 dqm->ops.uninitialize = uninitialize_nocpsch; 1149 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 1150 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 1151 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1152 break; 1153 case KFD_SCHED_POLICY_NO_HWS: 1154 /* initialize dqm for no cp scheduling */ 1155 dqm->ops.start = start_nocpsch; 1156 dqm->ops.stop = stop_nocpsch; 1157 dqm->ops.create_queue = create_queue_nocpsch; 1158 dqm->ops.destroy_queue = destroy_queue_nocpsch; 1159 dqm->ops.update_queue = update_queue; 1160 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1161 dqm->ops.register_process = register_process_nocpsch; 1162 dqm->ops.unregister_process = unregister_process_nocpsch; 1163 dqm->ops.initialize = initialize_nocpsch; 1164 dqm->ops.uninitialize = uninitialize_nocpsch; 1165 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1166 break; 1167 default: 1168 BUG(); 1169 break; 1170 } 1171 1172 switch (dev->device_info->asic_family) { 1173 case CHIP_CARRIZO: 1174 device_queue_manager_init_vi(&dqm->ops_asic_specific); 1175 break; 1176 1177 case CHIP_KAVERI: 1178 device_queue_manager_init_cik(&dqm->ops_asic_specific); 1179 break; 1180 } 1181 1182 if (dqm->ops.initialize(dqm) != 0) { 1183 kfree(dqm); 1184 return NULL; 1185 } 1186 1187 return dqm; 1188 } 1189 1190 void device_queue_manager_uninit(struct device_queue_manager *dqm) 1191 { 1192 BUG_ON(!dqm); 1193 1194 dqm->ops.uninitialize(dqm); 1195 kfree(dqm); 1196 } 1197