1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/slab.h> 25 #include <linux/mutex.h> 26 #include "kfd_device_queue_manager.h" 27 #include "kfd_kernel_queue.h" 28 #include "kfd_priv.h" 29 #include "kfd_pm4_headers.h" 30 #include "kfd_pm4_opcodes.h" 31 32 static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, 33 unsigned int buffer_size_bytes) 34 { 35 unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t); 36 37 BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes); 38 *wptr = temp; 39 } 40 41 static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size) 42 { 43 union PM4_MES_TYPE_3_HEADER header; 44 45 header.u32all = 0; 46 header.opcode = opcode; 47 header.count = packet_size/sizeof(uint32_t) - 2; 48 header.type = PM4_TYPE_3; 49 50 return header.u32all; 51 } 52 53 static void pm_calc_rlib_size(struct packet_manager *pm, 54 unsigned int *rlib_size, 55 bool *over_subscription) 56 { 57 unsigned int process_count, queue_count; 58 59 BUG_ON(!pm || !rlib_size || !over_subscription); 60 61 process_count = pm->dqm->processes_count; 62 queue_count = pm->dqm->queue_count; 63 64 /* check if there is over subscription*/ 65 *over_subscription = false; 66 if ((process_count > 1) || 67 queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) { 68 *over_subscription = true; 69 pr_debug("kfd: over subscribed runlist\n"); 70 } 71 72 /* calculate run list ib allocation size */ 73 *rlib_size = process_count * sizeof(struct pm4_map_process) + 74 queue_count * sizeof(struct pm4_map_queues); 75 76 /* 77 * Increase the allocation size in case we need a chained run list 78 * when over subscription 79 */ 80 if (*over_subscription) 81 *rlib_size += sizeof(struct pm4_runlist); 82 83 pr_debug("kfd: runlist ib size %d\n", *rlib_size); 84 } 85 86 static int pm_allocate_runlist_ib(struct packet_manager *pm, 87 unsigned int **rl_buffer, 88 uint64_t *rl_gpu_buffer, 89 unsigned int *rl_buffer_size, 90 bool *is_over_subscription) 91 { 92 int retval; 93 94 BUG_ON(!pm); 95 BUG_ON(pm->allocated == true); 96 BUG_ON(is_over_subscription == NULL); 97 98 pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription); 99 100 retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd, 101 *rl_buffer_size, 102 PAGE_SIZE, 103 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 104 (struct kgd_mem **) &pm->ib_buffer_obj); 105 106 if (retval != 0) { 107 pr_err("kfd: failed to allocate runlist IB\n"); 108 return retval; 109 } 110 111 *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr; 112 *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr; 113 114 memset(*rl_buffer, 0, *rl_buffer_size); 115 pm->allocated = true; 116 return retval; 117 } 118 119 static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer, 120 uint64_t ib, size_t ib_size_in_dwords, bool chain) 121 { 122 struct pm4_runlist *packet; 123 124 BUG_ON(!pm || !buffer || !ib); 125 126 packet = (struct pm4_runlist *)buffer; 127 128 memset(buffer, 0, sizeof(struct pm4_runlist)); 129 packet->header.u32all = build_pm4_header(IT_RUN_LIST, 130 sizeof(struct pm4_runlist)); 131 132 packet->bitfields4.ib_size = ib_size_in_dwords; 133 packet->bitfields4.chain = chain ? 1 : 0; 134 packet->bitfields4.offload_polling = 0; 135 packet->bitfields4.valid = 1; 136 packet->ordinal2 = lower_32_bits(ib); 137 packet->bitfields3.ib_base_hi = upper_32_bits(ib); 138 139 return 0; 140 } 141 142 static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer, 143 struct qcm_process_device *qpd) 144 { 145 struct pm4_map_process *packet; 146 struct queue *cur; 147 uint32_t num_queues; 148 149 BUG_ON(!pm || !buffer || !qpd); 150 151 packet = (struct pm4_map_process *)buffer; 152 153 pr_debug("kfd: In func %s\n", __func__); 154 155 memset(buffer, 0, sizeof(struct pm4_map_process)); 156 157 packet->header.u32all = build_pm4_header(IT_MAP_PROCESS, 158 sizeof(struct pm4_map_process)); 159 packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0; 160 packet->bitfields2.process_quantum = 1; 161 packet->bitfields2.pasid = qpd->pqm->process->pasid; 162 packet->bitfields3.page_table_base = qpd->page_table_base; 163 packet->bitfields10.gds_size = qpd->gds_size; 164 packet->bitfields10.num_gws = qpd->num_gws; 165 packet->bitfields10.num_oac = qpd->num_oac; 166 num_queues = 0; 167 list_for_each_entry(cur, &qpd->queues_list, list) 168 num_queues++; 169 packet->bitfields10.num_queues = num_queues; 170 171 packet->sh_mem_config = qpd->sh_mem_config; 172 packet->sh_mem_bases = qpd->sh_mem_bases; 173 packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base; 174 packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit; 175 176 packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area); 177 packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area); 178 179 return 0; 180 } 181 182 static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer, 183 struct queue *q) 184 { 185 struct pm4_map_queues *packet; 186 187 BUG_ON(!pm || !buffer || !q); 188 189 pr_debug("kfd: In func %s\n", __func__); 190 191 packet = (struct pm4_map_queues *)buffer; 192 memset(buffer, 0, sizeof(struct pm4_map_queues)); 193 194 packet->header.u32all = build_pm4_header(IT_MAP_QUEUES, 195 sizeof(struct pm4_map_queues)); 196 packet->bitfields2.alloc_format = 197 alloc_format__mes_map_queues__one_per_pipe; 198 packet->bitfields2.num_queues = 1; 199 packet->bitfields2.queue_sel = 200 queue_sel__mes_map_queues__map_to_hws_determined_queue_slots; 201 202 packet->bitfields2.vidmem = (q->properties.is_interop) ? 203 vidmem__mes_map_queues__uses_video_memory : 204 vidmem__mes_map_queues__uses_no_video_memory; 205 206 switch (q->properties.type) { 207 case KFD_QUEUE_TYPE_COMPUTE: 208 case KFD_QUEUE_TYPE_DIQ: 209 packet->bitfields2.engine_sel = 210 engine_sel__mes_map_queues__compute; 211 break; 212 case KFD_QUEUE_TYPE_SDMA: 213 packet->bitfields2.engine_sel = 214 engine_sel__mes_map_queues__sdma0; 215 break; 216 default: 217 BUG(); 218 break; 219 } 220 221 packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset = 222 q->properties.doorbell_off; 223 224 packet->mes_map_queues_ordinals[0].mqd_addr_lo = 225 lower_32_bits(q->gart_mqd_addr); 226 227 packet->mes_map_queues_ordinals[0].mqd_addr_hi = 228 upper_32_bits(q->gart_mqd_addr); 229 230 packet->mes_map_queues_ordinals[0].wptr_addr_lo = 231 lower_32_bits((uint64_t)q->properties.write_ptr); 232 233 packet->mes_map_queues_ordinals[0].wptr_addr_hi = 234 upper_32_bits((uint64_t)q->properties.write_ptr); 235 236 return 0; 237 } 238 239 static int pm_create_runlist_ib(struct packet_manager *pm, 240 struct list_head *queues, 241 uint64_t *rl_gpu_addr, 242 size_t *rl_size_bytes) 243 { 244 unsigned int alloc_size_bytes; 245 unsigned int *rl_buffer, rl_wptr, i; 246 int retval, proccesses_mapped; 247 struct device_process_node *cur; 248 struct qcm_process_device *qpd; 249 struct queue *q; 250 struct kernel_queue *kq; 251 bool is_over_subscription; 252 253 BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr); 254 255 rl_wptr = retval = proccesses_mapped = 0; 256 257 retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr, 258 &alloc_size_bytes, &is_over_subscription); 259 if (retval != 0) 260 return retval; 261 262 *rl_size_bytes = alloc_size_bytes; 263 264 pr_debug("kfd: In func %s\n", __func__); 265 pr_debug("kfd: building runlist ib process count: %d queues count %d\n", 266 pm->dqm->processes_count, pm->dqm->queue_count); 267 268 /* build the run list ib packet */ 269 list_for_each_entry(cur, queues, list) { 270 qpd = cur->qpd; 271 /* build map process packet */ 272 if (proccesses_mapped >= pm->dqm->processes_count) { 273 pr_debug("kfd: not enough space left in runlist IB\n"); 274 pm_release_ib(pm); 275 return -ENOMEM; 276 } 277 retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd); 278 if (retval != 0) 279 return retval; 280 proccesses_mapped++; 281 inc_wptr(&rl_wptr, sizeof(struct pm4_map_process), 282 alloc_size_bytes); 283 284 list_for_each_entry(kq, &qpd->priv_queue_list, list) { 285 if (kq->queue->properties.is_active != true) 286 continue; 287 retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], 288 kq->queue); 289 if (retval != 0) 290 return retval; 291 inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), 292 alloc_size_bytes); 293 } 294 295 list_for_each_entry(q, &qpd->queues_list, list) { 296 if (q->properties.is_active != true) 297 continue; 298 retval = pm_create_map_queue(pm, 299 &rl_buffer[rl_wptr], q); 300 if (retval != 0) 301 return retval; 302 inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), 303 alloc_size_bytes); 304 } 305 } 306 307 pr_debug("kfd: finished map process and queues to runlist\n"); 308 309 if (is_over_subscription) 310 pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr, 311 alloc_size_bytes / sizeof(uint32_t), true); 312 313 for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++) 314 pr_debug("0x%2X ", rl_buffer[i]); 315 pr_debug("\n"); 316 317 return 0; 318 } 319 320 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) 321 { 322 BUG_ON(!dqm); 323 324 pm->dqm = dqm; 325 mutex_init(&pm->lock); 326 pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); 327 if (pm->priv_queue == NULL) { 328 mutex_destroy(&pm->lock); 329 return -ENOMEM; 330 } 331 pm->allocated = false; 332 333 return 0; 334 } 335 336 void pm_uninit(struct packet_manager *pm) 337 { 338 BUG_ON(!pm); 339 340 mutex_destroy(&pm->lock); 341 kernel_queue_uninit(pm->priv_queue); 342 } 343 344 int pm_send_set_resources(struct packet_manager *pm, 345 struct scheduling_resources *res) 346 { 347 struct pm4_set_resources *packet; 348 349 BUG_ON(!pm || !res); 350 351 pr_debug("kfd: In func %s\n", __func__); 352 353 mutex_lock(&pm->lock); 354 pm->priv_queue->acquire_packet_buffer(pm->priv_queue, 355 sizeof(*packet) / sizeof(uint32_t), 356 (unsigned int **)&packet); 357 if (packet == NULL) { 358 mutex_unlock(&pm->lock); 359 pr_err("kfd: failed to allocate buffer on kernel queue\n"); 360 return -ENOMEM; 361 } 362 363 memset(packet, 0, sizeof(struct pm4_set_resources)); 364 packet->header.u32all = build_pm4_header(IT_SET_RESOURCES, 365 sizeof(struct pm4_set_resources)); 366 367 packet->bitfields2.queue_type = 368 queue_type__mes_set_resources__hsa_interface_queue_hiq; 369 packet->bitfields2.vmid_mask = res->vmid_mask; 370 packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY; 371 packet->bitfields7.oac_mask = res->oac_mask; 372 packet->bitfields8.gds_heap_base = res->gds_heap_base; 373 packet->bitfields8.gds_heap_size = res->gds_heap_size; 374 375 packet->gws_mask_lo = lower_32_bits(res->gws_mask); 376 packet->gws_mask_hi = upper_32_bits(res->gws_mask); 377 378 packet->queue_mask_lo = lower_32_bits(res->queue_mask); 379 packet->queue_mask_hi = upper_32_bits(res->queue_mask); 380 381 pm->priv_queue->submit_packet(pm->priv_queue); 382 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); 383 384 mutex_unlock(&pm->lock); 385 386 return 0; 387 } 388 389 int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues) 390 { 391 uint64_t rl_gpu_ib_addr; 392 uint32_t *rl_buffer; 393 size_t rl_ib_size, packet_size_dwords; 394 int retval; 395 396 BUG_ON(!pm || !dqm_queues); 397 398 retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr, 399 &rl_ib_size); 400 if (retval != 0) 401 goto fail_create_runlist_ib; 402 403 pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr); 404 405 packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t); 406 mutex_lock(&pm->lock); 407 408 retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, 409 packet_size_dwords, &rl_buffer); 410 if (retval != 0) 411 goto fail_acquire_packet_buffer; 412 413 retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr, 414 rl_ib_size / sizeof(uint32_t), false); 415 if (retval != 0) 416 goto fail_create_runlist; 417 418 pm->priv_queue->submit_packet(pm->priv_queue); 419 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); 420 421 mutex_unlock(&pm->lock); 422 423 return retval; 424 425 fail_create_runlist: 426 pm->priv_queue->rollback_packet(pm->priv_queue); 427 fail_acquire_packet_buffer: 428 mutex_unlock(&pm->lock); 429 fail_create_runlist_ib: 430 if (pm->allocated == true) 431 pm_release_ib(pm); 432 return retval; 433 } 434 435 int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, 436 uint32_t fence_value) 437 { 438 int retval; 439 struct pm4_query_status *packet; 440 441 BUG_ON(!pm || !fence_address); 442 443 mutex_lock(&pm->lock); 444 retval = pm->priv_queue->acquire_packet_buffer( 445 pm->priv_queue, 446 sizeof(struct pm4_query_status) / sizeof(uint32_t), 447 (unsigned int **)&packet); 448 if (retval != 0) 449 goto fail_acquire_packet_buffer; 450 451 packet->header.u32all = build_pm4_header(IT_QUERY_STATUS, 452 sizeof(struct pm4_query_status)); 453 454 packet->bitfields2.context_id = 0; 455 packet->bitfields2.interrupt_sel = 456 interrupt_sel__mes_query_status__completion_status; 457 packet->bitfields2.command = 458 command__mes_query_status__fence_only_after_write_ack; 459 460 packet->addr_hi = upper_32_bits((uint64_t)fence_address); 461 packet->addr_lo = lower_32_bits((uint64_t)fence_address); 462 packet->data_hi = upper_32_bits((uint64_t)fence_value); 463 packet->data_lo = lower_32_bits((uint64_t)fence_value); 464 465 pm->priv_queue->submit_packet(pm->priv_queue); 466 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); 467 mutex_unlock(&pm->lock); 468 469 return 0; 470 471 fail_acquire_packet_buffer: 472 mutex_unlock(&pm->lock); 473 return retval; 474 } 475 476 int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, 477 enum kfd_preempt_type_filter mode, 478 uint32_t filter_param, bool reset, 479 unsigned int sdma_engine) 480 { 481 int retval; 482 uint32_t *buffer; 483 struct pm4_unmap_queues *packet; 484 485 BUG_ON(!pm); 486 487 mutex_lock(&pm->lock); 488 retval = pm->priv_queue->acquire_packet_buffer( 489 pm->priv_queue, 490 sizeof(struct pm4_unmap_queues) / sizeof(uint32_t), 491 &buffer); 492 if (retval != 0) 493 goto err_acquire_packet_buffer; 494 495 packet = (struct pm4_unmap_queues *)buffer; 496 memset(buffer, 0, sizeof(struct pm4_unmap_queues)); 497 498 packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES, 499 sizeof(struct pm4_unmap_queues)); 500 switch (type) { 501 case KFD_QUEUE_TYPE_COMPUTE: 502 case KFD_QUEUE_TYPE_DIQ: 503 packet->bitfields2.engine_sel = 504 engine_sel__mes_unmap_queues__compute; 505 break; 506 case KFD_QUEUE_TYPE_SDMA: 507 packet->bitfields2.engine_sel = 508 engine_sel__mes_unmap_queues__sdma0 + sdma_engine; 509 break; 510 default: 511 BUG(); 512 break; 513 } 514 515 if (reset) 516 packet->bitfields2.action = 517 action__mes_unmap_queues__reset_queues; 518 else 519 packet->bitfields2.action = 520 action__mes_unmap_queues__preempt_queues; 521 522 switch (mode) { 523 case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: 524 packet->bitfields2.queue_sel = 525 queue_sel__mes_unmap_queues__perform_request_on_specified_queues; 526 packet->bitfields2.num_queues = 1; 527 packet->bitfields3b.doorbell_offset0 = filter_param; 528 break; 529 case KFD_PREEMPT_TYPE_FILTER_BY_PASID: 530 packet->bitfields2.queue_sel = 531 queue_sel__mes_unmap_queues__perform_request_on_pasid_queues; 532 packet->bitfields3a.pasid = filter_param; 533 break; 534 case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES: 535 packet->bitfields2.queue_sel = 536 queue_sel__mes_unmap_queues__perform_request_on_all_active_queues; 537 break; 538 default: 539 BUG(); 540 break; 541 }; 542 543 pm->priv_queue->submit_packet(pm->priv_queue); 544 pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); 545 546 mutex_unlock(&pm->lock); 547 return 0; 548 549 err_acquire_packet_buffer: 550 mutex_unlock(&pm->lock); 551 return retval; 552 } 553 554 void pm_release_ib(struct packet_manager *pm) 555 { 556 BUG_ON(!pm); 557 558 mutex_lock(&pm->lock); 559 if (pm->allocated) { 560 kfd2kgd->free_mem(pm->dqm->dev->kgd, 561 (struct kgd_mem *) pm->ib_buffer_obj); 562 pm->allocated = false; 563 } 564 mutex_unlock(&pm->lock); 565 } 566