1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2016-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "habanalabs.h" 9 10 #include <linux/slab.h> 11 12 /** 13 * struct hl_eqe_work - This structure is used to schedule work of EQ 14 * entry and cpucp_reset event 15 * 16 * @eq_work: workqueue object to run when EQ entry is received 17 * @hdev: pointer to device structure 18 * @eq_entry: copy of the EQ entry 19 */ 20 struct hl_eqe_work { 21 struct work_struct eq_work; 22 struct hl_device *hdev; 23 struct hl_eq_entry eq_entry; 24 }; 25 26 /** 27 * hl_cq_inc_ptr - increment ci or pi of cq 28 * 29 * @ptr: the current ci or pi value of the completion queue 30 * 31 * Increment ptr by 1. If it reaches the number of completion queue 32 * entries, set it to 0 33 */ 34 inline u32 hl_cq_inc_ptr(u32 ptr) 35 { 36 ptr++; 37 if (unlikely(ptr == HL_CQ_LENGTH)) 38 ptr = 0; 39 return ptr; 40 } 41 42 /** 43 * hl_eq_inc_ptr - increment ci of eq 44 * 45 * @ptr: the current ci value of the event queue 46 * 47 * Increment ptr by 1. If it reaches the number of event queue 48 * entries, set it to 0 49 */ 50 static inline u32 hl_eq_inc_ptr(u32 ptr) 51 { 52 ptr++; 53 if (unlikely(ptr == HL_EQ_LENGTH)) 54 ptr = 0; 55 return ptr; 56 } 57 58 static void irq_handle_eqe(struct work_struct *work) 59 { 60 struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work, 61 eq_work); 62 struct hl_device *hdev = eqe_work->hdev; 63 64 hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry); 65 66 kfree(eqe_work); 67 } 68 69 /** 70 * job_finish - queue job finish work 71 * 72 * @hdev: pointer to device structure 73 * @cs_seq: command submission sequence 74 * @cq: completion queue 75 * @timestamp: interrupt timestamp 76 * 77 */ 78 static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp) 79 { 80 struct hl_hw_queue *queue; 81 struct hl_cs_job *job; 82 83 queue = &hdev->kernel_queues[cq->hw_queue_id]; 84 job = queue->shadow_queue[hl_pi_2_offset(cs_seq)]; 85 job->timestamp = timestamp; 86 queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work); 87 88 atomic_inc(&queue->ci); 89 } 90 91 /** 92 * cs_finish - queue all cs jobs finish work 93 * 94 * @hdev: pointer to device structure 95 * @cs_seq: command submission sequence 96 * @timestamp: interrupt timestamp 97 * 98 */ 99 static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp) 100 { 101 struct asic_fixed_properties *prop = &hdev->asic_prop; 102 struct hl_hw_queue *queue; 103 struct hl_cs *cs; 104 struct hl_cs_job *job; 105 106 cs = hdev->shadow_cs_queue[cs_seq & (prop->max_pending_cs - 1)]; 107 if (!cs) { 108 dev_warn(hdev->dev, 109 "No pointer to CS in shadow array at index %d\n", 110 cs_seq); 111 return; 112 } 113 114 list_for_each_entry(job, &cs->job_list, cs_node) { 115 queue = &hdev->kernel_queues[job->hw_queue_id]; 116 atomic_inc(&queue->ci); 117 } 118 119 cs->completion_timestamp = timestamp; 120 queue_work(hdev->cs_cmplt_wq, &cs->finish_work); 121 } 122 123 /** 124 * hl_irq_handler_cq - irq handler for completion queue 125 * 126 * @irq: irq number 127 * @arg: pointer to completion queue structure 128 * 129 */ 130 irqreturn_t hl_irq_handler_cq(int irq, void *arg) 131 { 132 struct hl_cq *cq = arg; 133 struct hl_device *hdev = cq->hdev; 134 bool shadow_index_valid, entry_ready; 135 u16 shadow_index; 136 struct hl_cq_entry *cq_entry, *cq_base; 137 ktime_t timestamp = ktime_get(); 138 139 if (hdev->disabled) { 140 dev_dbg(hdev->dev, 141 "Device disabled but received IRQ %d for CQ %d\n", 142 irq, cq->hw_queue_id); 143 return IRQ_HANDLED; 144 } 145 146 cq_base = cq->kernel_address; 147 148 while (1) { 149 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci]; 150 151 entry_ready = !!FIELD_GET(CQ_ENTRY_READY_MASK, 152 le32_to_cpu(cq_entry->data)); 153 if (!entry_ready) 154 break; 155 156 /* Make sure we read CQ entry contents after we've 157 * checked the ownership bit. 158 */ 159 dma_rmb(); 160 161 shadow_index_valid = 162 !!FIELD_GET(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 163 le32_to_cpu(cq_entry->data)); 164 165 shadow_index = FIELD_GET(CQ_ENTRY_SHADOW_INDEX_MASK, 166 le32_to_cpu(cq_entry->data)); 167 168 /* 169 * CQ interrupt handler has 2 modes of operation: 170 * 1. Interrupt per CS completion: (Single CQ for all queues) 171 * CQ entry represents a completed CS 172 * 173 * 2. Interrupt per CS job completion in queue: (CQ per queue) 174 * CQ entry represents a completed job in a certain queue 175 */ 176 if (shadow_index_valid && !hdev->disabled) { 177 if (hdev->asic_prop.completion_mode == 178 HL_COMPLETION_MODE_CS) 179 cs_finish(hdev, shadow_index, timestamp); 180 else 181 job_finish(hdev, shadow_index, cq, timestamp); 182 } 183 184 /* Clear CQ entry ready bit */ 185 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) & 186 ~CQ_ENTRY_READY_MASK); 187 188 cq->ci = hl_cq_inc_ptr(cq->ci); 189 190 /* Increment free slots */ 191 atomic_inc(&cq->free_slots_cnt); 192 } 193 194 return IRQ_HANDLED; 195 } 196 197 /* 198 * hl_ts_free_objects - handler of the free objects workqueue. 199 * This function should put refcount to objects that the registration node 200 * took refcount to them. 201 * @work: workqueue object pointer 202 */ 203 static void hl_ts_free_objects(struct work_struct *work) 204 { 205 struct timestamp_reg_work_obj *job = 206 container_of(work, struct timestamp_reg_work_obj, free_obj); 207 struct timestamp_reg_free_node *free_obj, *temp_free_obj; 208 struct list_head *free_list_head = job->free_obj_head; 209 struct hl_device *hdev = job->hdev; 210 211 list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) { 212 dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n", 213 free_obj->buf, 214 free_obj->cq_cb); 215 216 hl_mmap_mem_buf_put(free_obj->buf); 217 hl_cb_put(free_obj->cq_cb); 218 kfree(free_obj); 219 } 220 221 kfree(free_list_head); 222 kfree(job); 223 } 224 225 /* 226 * This function called with spin_lock of wait_list_lock taken 227 * This function will set timestamp and delete the registration node from the 228 * wait_list_lock. 229 * and since we're protected with spin_lock here, so we cannot just put the refcount 230 * for the objects here, since the release function may be called and it's also a long 231 * logic (which might sleep also) that cannot be handled in irq context. 232 * so here we'll be filling a list with nodes of "put" jobs and then will send this 233 * list to a dedicated workqueue to do the actual put. 234 */ 235 static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend, 236 struct list_head **free_list, ktime_t now) 237 { 238 struct timestamp_reg_free_node *free_node; 239 u64 timestamp; 240 241 if (!(*free_list)) { 242 /* Alloc/Init the timestamp registration free objects list */ 243 *free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC); 244 if (!(*free_list)) 245 return -ENOMEM; 246 247 INIT_LIST_HEAD(*free_list); 248 } 249 250 free_node = kmalloc(sizeof(*free_node), GFP_ATOMIC); 251 if (!free_node) 252 return -ENOMEM; 253 254 timestamp = ktime_to_ns(now); 255 256 *pend->ts_reg_info.timestamp_kernel_addr = timestamp; 257 258 dev_dbg(hdev->dev, "Timestamp is set to ts cb address (%p), ts: 0x%llx\n", 259 pend->ts_reg_info.timestamp_kernel_addr, 260 *(u64 *)pend->ts_reg_info.timestamp_kernel_addr); 261 262 list_del(&pend->wait_list_node); 263 264 /* Mark kernel CB node as free */ 265 pend->ts_reg_info.in_use = 0; 266 267 /* Putting the refcount for ts_buff and cq_cb objects will be handled 268 * in workqueue context, just add job to free_list. 269 */ 270 free_node->buf = pend->ts_reg_info.buf; 271 free_node->cq_cb = pend->ts_reg_info.cq_cb; 272 list_add(&free_node->free_objects_node, *free_list); 273 274 return 0; 275 } 276 277 static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interrupt *intr) 278 { 279 struct hl_user_pending_interrupt *pend, *temp_pend; 280 struct list_head *ts_reg_free_list_head = NULL; 281 struct timestamp_reg_work_obj *job; 282 bool reg_node_handle_fail = false; 283 ktime_t now = ktime_get(); 284 int rc; 285 286 /* For registration nodes: 287 * As part of handling the registration nodes, we should put refcount to 288 * some objects. the problem is that we cannot do that under spinlock 289 * or in irq handler context at all (since release functions are long and 290 * might sleep), so we will need to handle that part in workqueue context. 291 * To avoid handling kmalloc failure which compels us rolling back actions 292 * and move nodes hanged on the free list back to the interrupt wait list 293 * we always alloc the job of the WQ at the beginning. 294 */ 295 job = kmalloc(sizeof(*job), GFP_ATOMIC); 296 if (!job) 297 return; 298 299 spin_lock(&intr->wait_list_lock); 300 list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, wait_list_node) { 301 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) || 302 !pend->cq_kernel_addr) { 303 if (pend->ts_reg_info.buf) { 304 if (!reg_node_handle_fail) { 305 rc = handle_registration_node(hdev, pend, 306 &ts_reg_free_list_head, now); 307 if (rc) 308 reg_node_handle_fail = true; 309 } 310 } else { 311 /* Handle wait target value node */ 312 pend->fence.timestamp = now; 313 complete_all(&pend->fence.completion); 314 } 315 } 316 } 317 spin_unlock(&intr->wait_list_lock); 318 319 if (ts_reg_free_list_head) { 320 INIT_WORK(&job->free_obj, hl_ts_free_objects); 321 job->free_obj_head = ts_reg_free_list_head; 322 job->hdev = hdev; 323 queue_work(hdev->ts_free_obj_wq, &job->free_obj); 324 } else { 325 kfree(job); 326 } 327 } 328 329 /** 330 * hl_irq_handler_user_interrupt - irq handler for user interrupts 331 * 332 * @irq: irq number 333 * @arg: pointer to user interrupt structure 334 * 335 */ 336 irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg) 337 { 338 struct hl_user_interrupt *user_int = arg; 339 struct hl_device *hdev = user_int->hdev; 340 341 switch (user_int->type) { 342 case HL_USR_INTERRUPT_CQ: 343 handle_user_interrupt(hdev, &hdev->common_user_cq_interrupt); 344 345 /* Handle user cq interrupt registered on this specific irq */ 346 handle_user_interrupt(hdev, user_int); 347 break; 348 case HL_USR_INTERRUPT_DECODER: 349 handle_user_interrupt(hdev, &hdev->common_decoder_interrupt); 350 351 /* Handle decoder interrupt registered on this specific irq */ 352 handle_user_interrupt(hdev, user_int); 353 break; 354 default: 355 break; 356 } 357 358 return IRQ_HANDLED; 359 } 360 361 /** 362 * hl_irq_handler_default - default irq handler 363 * 364 * @irq: irq number 365 * @arg: pointer to user interrupt structure 366 * 367 */ 368 irqreturn_t hl_irq_handler_default(int irq, void *arg) 369 { 370 struct hl_user_interrupt *user_interrupt = arg; 371 struct hl_device *hdev = user_interrupt->hdev; 372 u32 interrupt_id = user_interrupt->interrupt_id; 373 374 dev_err(hdev->dev, "got invalid user interrupt %u", interrupt_id); 375 376 return IRQ_HANDLED; 377 } 378 379 /** 380 * hl_irq_handler_eq - irq handler for event queue 381 * 382 * @irq: irq number 383 * @arg: pointer to event queue structure 384 * 385 */ 386 irqreturn_t hl_irq_handler_eq(int irq, void *arg) 387 { 388 struct hl_eq *eq = arg; 389 struct hl_device *hdev = eq->hdev; 390 struct hl_eq_entry *eq_entry; 391 struct hl_eq_entry *eq_base; 392 struct hl_eqe_work *handle_eqe_work; 393 bool entry_ready; 394 u32 cur_eqe; 395 u16 cur_eqe_index; 396 397 eq_base = eq->kernel_address; 398 399 while (1) { 400 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl); 401 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe); 402 403 if (!entry_ready) 404 break; 405 406 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe); 407 if ((hdev->event_queue.check_eqe_index) && 408 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) 409 != cur_eqe_index)) { 410 dev_dbg(hdev->dev, 411 "EQE 0x%x in queue is ready but index does not match %d!=%d", 412 eq_base[eq->ci].hdr.ctl, 413 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK), 414 cur_eqe_index); 415 break; 416 } 417 418 eq->prev_eqe_index++; 419 420 eq_entry = &eq_base[eq->ci]; 421 422 /* 423 * Make sure we read EQ entry contents after we've 424 * checked the ownership bit. 425 */ 426 dma_rmb(); 427 428 if (hdev->disabled && !hdev->reset_info.in_compute_reset) { 429 dev_warn(hdev->dev, "Device disabled but received an EQ event\n"); 430 goto skip_irq; 431 } 432 433 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC); 434 if (handle_eqe_work) { 435 INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe); 436 handle_eqe_work->hdev = hdev; 437 438 memcpy(&handle_eqe_work->eq_entry, eq_entry, 439 sizeof(*eq_entry)); 440 441 queue_work(hdev->eq_wq, &handle_eqe_work->eq_work); 442 } 443 skip_irq: 444 /* Clear EQ entry ready bit */ 445 eq_entry->hdr.ctl = 446 cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) & 447 ~EQ_CTL_READY_MASK); 448 449 eq->ci = hl_eq_inc_ptr(eq->ci); 450 451 hdev->asic_funcs->update_eq_ci(hdev, eq->ci); 452 } 453 454 return IRQ_HANDLED; 455 } 456 457 /** 458 * hl_irq_handler_dec_abnrm - Decoder error interrupt handler 459 * @irq: IRQ number 460 * @arg: pointer to decoder structure. 461 */ 462 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg) 463 { 464 struct hl_dec *dec = arg; 465 466 schedule_work(&dec->completion_abnrm_work); 467 468 return IRQ_HANDLED; 469 } 470 471 /** 472 * hl_cq_init - main initialization function for an cq object 473 * 474 * @hdev: pointer to device structure 475 * @q: pointer to cq structure 476 * @hw_queue_id: The H/W queue ID this completion queue belongs to 477 * HL_INVALID_QUEUE if cq is not attached to any specific queue 478 * 479 * Allocate dma-able memory for the completion queue and initialize fields 480 * Returns 0 on success 481 */ 482 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id) 483 { 484 void *p; 485 486 p = hl_asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES, &q->bus_address, 487 GFP_KERNEL | __GFP_ZERO); 488 if (!p) 489 return -ENOMEM; 490 491 q->hdev = hdev; 492 q->kernel_address = p; 493 q->hw_queue_id = hw_queue_id; 494 q->ci = 0; 495 q->pi = 0; 496 497 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH); 498 499 return 0; 500 } 501 502 /** 503 * hl_cq_fini - destroy completion queue 504 * 505 * @hdev: pointer to device structure 506 * @q: pointer to cq structure 507 * 508 * Free the completion queue memory 509 */ 510 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q) 511 { 512 hl_asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES, q->kernel_address, q->bus_address); 513 } 514 515 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q) 516 { 517 q->ci = 0; 518 q->pi = 0; 519 520 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH); 521 522 /* 523 * It's not enough to just reset the PI/CI because the H/W may have 524 * written valid completion entries before it was halted and therefore 525 * we need to clean the actual queues so we won't process old entries 526 * when the device is operational again 527 */ 528 529 memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES); 530 } 531 532 /** 533 * hl_eq_init - main initialization function for an event queue object 534 * 535 * @hdev: pointer to device structure 536 * @q: pointer to eq structure 537 * 538 * Allocate dma-able memory for the event queue and initialize fields 539 * Returns 0 on success 540 */ 541 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q) 542 { 543 void *p; 544 545 p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_EQ_SIZE_IN_BYTES, &q->bus_address); 546 if (!p) 547 return -ENOMEM; 548 549 q->hdev = hdev; 550 q->kernel_address = p; 551 q->ci = 0; 552 q->prev_eqe_index = 0; 553 554 return 0; 555 } 556 557 /** 558 * hl_eq_fini - destroy event queue 559 * 560 * @hdev: pointer to device structure 561 * @q: pointer to eq structure 562 * 563 * Free the event queue memory 564 */ 565 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q) 566 { 567 flush_workqueue(hdev->eq_wq); 568 569 hl_cpu_accessible_dma_pool_free(hdev, HL_EQ_SIZE_IN_BYTES, q->kernel_address); 570 } 571 572 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q) 573 { 574 q->ci = 0; 575 q->prev_eqe_index = 0; 576 577 /* 578 * It's not enough to just reset the PI/CI because the H/W may have 579 * written valid completion entries before it was halted and therefore 580 * we need to clean the actual queues so we won't process old entries 581 * when the device is operational again 582 */ 583 584 memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES); 585 } 586