1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include "habanalabs.h"
9 
10 #include <linux/slab.h>
11 
12 /**
13  * struct hl_eqe_work - This structure is used to schedule work of EQ
14  *                      entry and cpucp_reset event
15  *
16  * @eq_work:          workqueue object to run when EQ entry is received
17  * @hdev:             pointer to device structure
18  * @eq_entry:         copy of the EQ entry
19  */
20 struct hl_eqe_work {
21 	struct work_struct	eq_work;
22 	struct hl_device	*hdev;
23 	struct hl_eq_entry	eq_entry;
24 };
25 
26 /**
27  * hl_cq_inc_ptr - increment ci or pi of cq
28  *
29  * @ptr: the current ci or pi value of the completion queue
30  *
31  * Increment ptr by 1. If it reaches the number of completion queue
32  * entries, set it to 0
33  */
hl_cq_inc_ptr(u32 ptr)34 inline u32 hl_cq_inc_ptr(u32 ptr)
35 {
36 	ptr++;
37 	if (unlikely(ptr == HL_CQ_LENGTH))
38 		ptr = 0;
39 	return ptr;
40 }
41 
42 /**
43  * hl_eq_inc_ptr - increment ci of eq
44  *
45  * @ptr: the current ci value of the event queue
46  *
47  * Increment ptr by 1. If it reaches the number of event queue
48  * entries, set it to 0
49  */
hl_eq_inc_ptr(u32 ptr)50 static inline u32 hl_eq_inc_ptr(u32 ptr)
51 {
52 	ptr++;
53 	if (unlikely(ptr == HL_EQ_LENGTH))
54 		ptr = 0;
55 	return ptr;
56 }
57 
irq_handle_eqe(struct work_struct * work)58 static void irq_handle_eqe(struct work_struct *work)
59 {
60 	struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
61 							eq_work);
62 	struct hl_device *hdev = eqe_work->hdev;
63 
64 	hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
65 
66 	kfree(eqe_work);
67 }
68 
69 /**
70  * job_finish - queue job finish work
71  *
72  * @hdev: pointer to device structure
73  * @cs_seq: command submission sequence
74  * @cq: completion queue
75  * @timestamp: interrupt timestamp
76  *
77  */
job_finish(struct hl_device * hdev,u32 cs_seq,struct hl_cq * cq,ktime_t timestamp)78 static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp)
79 {
80 	struct hl_hw_queue *queue;
81 	struct hl_cs_job *job;
82 
83 	queue = &hdev->kernel_queues[cq->hw_queue_id];
84 	job = queue->shadow_queue[hl_pi_2_offset(cs_seq)];
85 	job->timestamp = timestamp;
86 	queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
87 
88 	atomic_inc(&queue->ci);
89 }
90 
91 /**
92  * cs_finish - queue all cs jobs finish work
93  *
94  * @hdev: pointer to device structure
95  * @cs_seq: command submission sequence
96  * @timestamp: interrupt timestamp
97  *
98  */
cs_finish(struct hl_device * hdev,u16 cs_seq,ktime_t timestamp)99 static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp)
100 {
101 	struct asic_fixed_properties *prop = &hdev->asic_prop;
102 	struct hl_hw_queue *queue;
103 	struct hl_cs *cs;
104 	struct hl_cs_job *job;
105 
106 	cs = hdev->shadow_cs_queue[cs_seq & (prop->max_pending_cs - 1)];
107 	if (!cs) {
108 		dev_warn(hdev->dev,
109 			"No pointer to CS in shadow array at index %d\n",
110 			cs_seq);
111 		return;
112 	}
113 
114 	list_for_each_entry(job, &cs->job_list, cs_node) {
115 		queue = &hdev->kernel_queues[job->hw_queue_id];
116 		atomic_inc(&queue->ci);
117 	}
118 
119 	cs->completion_timestamp = timestamp;
120 	queue_work(hdev->cs_cmplt_wq, &cs->finish_work);
121 }
122 
123 /**
124  * hl_irq_handler_cq - irq handler for completion queue
125  *
126  * @irq: irq number
127  * @arg: pointer to completion queue structure
128  *
129  */
hl_irq_handler_cq(int irq,void * arg)130 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
131 {
132 	struct hl_cq *cq = arg;
133 	struct hl_device *hdev = cq->hdev;
134 	bool shadow_index_valid, entry_ready;
135 	u16 shadow_index;
136 	struct hl_cq_entry *cq_entry, *cq_base;
137 	ktime_t timestamp = ktime_get();
138 
139 	if (hdev->disabled) {
140 		dev_dbg(hdev->dev,
141 			"Device disabled but received IRQ %d for CQ %d\n",
142 			irq, cq->hw_queue_id);
143 		return IRQ_HANDLED;
144 	}
145 
146 	cq_base = cq->kernel_address;
147 
148 	while (1) {
149 		cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
150 
151 		entry_ready = !!FIELD_GET(CQ_ENTRY_READY_MASK,
152 				le32_to_cpu(cq_entry->data));
153 		if (!entry_ready)
154 			break;
155 
156 		/* Make sure we read CQ entry contents after we've
157 		 * checked the ownership bit.
158 		 */
159 		dma_rmb();
160 
161 		shadow_index_valid =
162 			!!FIELD_GET(CQ_ENTRY_SHADOW_INDEX_VALID_MASK,
163 					le32_to_cpu(cq_entry->data));
164 
165 		shadow_index = FIELD_GET(CQ_ENTRY_SHADOW_INDEX_MASK,
166 				le32_to_cpu(cq_entry->data));
167 
168 		/*
169 		 * CQ interrupt handler has 2 modes of operation:
170 		 * 1. Interrupt per CS completion: (Single CQ for all queues)
171 		 *    CQ entry represents a completed CS
172 		 *
173 		 * 2. Interrupt per CS job completion in queue: (CQ per queue)
174 		 *    CQ entry represents a completed job in a certain queue
175 		 */
176 		if (shadow_index_valid && !hdev->disabled) {
177 			if (hdev->asic_prop.completion_mode ==
178 					HL_COMPLETION_MODE_CS)
179 				cs_finish(hdev, shadow_index, timestamp);
180 			else
181 				job_finish(hdev, shadow_index, cq, timestamp);
182 		}
183 
184 		/* Clear CQ entry ready bit */
185 		cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
186 						~CQ_ENTRY_READY_MASK);
187 
188 		cq->ci = hl_cq_inc_ptr(cq->ci);
189 
190 		/* Increment free slots */
191 		atomic_inc(&cq->free_slots_cnt);
192 	}
193 
194 	return IRQ_HANDLED;
195 }
196 
197 /*
198  * hl_ts_free_objects - handler of the free objects workqueue.
199  * This function should put refcount to objects that the registration node
200  * took refcount to them.
201  * @work: workqueue object pointer
202  */
hl_ts_free_objects(struct work_struct * work)203 static void hl_ts_free_objects(struct work_struct *work)
204 {
205 	struct timestamp_reg_work_obj *job =
206 			container_of(work, struct timestamp_reg_work_obj, free_obj);
207 	struct timestamp_reg_free_node *free_obj, *temp_free_obj;
208 	struct list_head *free_list_head = job->free_obj_head;
209 	struct hl_device *hdev = job->hdev;
210 
211 	list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
212 		dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n",
213 					free_obj->buf,
214 					free_obj->cq_cb);
215 
216 		hl_mmap_mem_buf_put(free_obj->buf);
217 		hl_cb_put(free_obj->cq_cb);
218 		kfree(free_obj);
219 	}
220 
221 	kfree(free_list_head);
222 	kfree(job);
223 }
224 
225 /*
226  * This function called with spin_lock of wait_list_lock taken
227  * This function will set timestamp and delete the registration node from the
228  * wait_list_lock.
229  * and since we're protected with spin_lock here, so we cannot just put the refcount
230  * for the objects here, since the release function may be called and it's also a long
231  * logic (which might sleep also) that cannot be handled in irq context.
232  * so here we'll be filling a list with nodes of "put" jobs and then will send this
233  * list to a dedicated workqueue to do the actual put.
234  */
handle_registration_node(struct hl_device * hdev,struct hl_user_pending_interrupt * pend,struct list_head ** free_list,ktime_t now)235 static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
236 						struct list_head **free_list, ktime_t now)
237 {
238 	struct timestamp_reg_free_node *free_node;
239 	u64 timestamp;
240 
241 	if (!(*free_list)) {
242 		/* Alloc/Init the timestamp registration free objects list */
243 		*free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
244 		if (!(*free_list))
245 			return -ENOMEM;
246 
247 		INIT_LIST_HEAD(*free_list);
248 	}
249 
250 	free_node = kmalloc(sizeof(*free_node), GFP_ATOMIC);
251 	if (!free_node)
252 		return -ENOMEM;
253 
254 	timestamp = ktime_to_ns(now);
255 
256 	*pend->ts_reg_info.timestamp_kernel_addr = timestamp;
257 
258 	dev_dbg(hdev->dev, "Timestamp is set to ts cb address (%p), ts: 0x%llx\n",
259 			pend->ts_reg_info.timestamp_kernel_addr,
260 			*(u64 *)pend->ts_reg_info.timestamp_kernel_addr);
261 
262 	list_del(&pend->wait_list_node);
263 
264 	/* Mark kernel CB node as free */
265 	pend->ts_reg_info.in_use = 0;
266 
267 	/* Putting the refcount for ts_buff and cq_cb objects will be handled
268 	 * in workqueue context, just add job to free_list.
269 	 */
270 	free_node->buf = pend->ts_reg_info.buf;
271 	free_node->cq_cb = pend->ts_reg_info.cq_cb;
272 	list_add(&free_node->free_objects_node, *free_list);
273 
274 	/* Mark TS record as free */
275 	pend->ts_reg_info.in_use = false;
276 
277 	return 0;
278 }
279 
handle_user_interrupt(struct hl_device * hdev,struct hl_user_interrupt * intr)280 static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interrupt *intr)
281 {
282 	struct hl_user_pending_interrupt *pend, *temp_pend;
283 	struct list_head *ts_reg_free_list_head = NULL;
284 	struct timestamp_reg_work_obj *job;
285 	bool reg_node_handle_fail = false;
286 	int rc;
287 
288 	/* For registration nodes:
289 	 * As part of handling the registration nodes, we should put refcount to
290 	 * some objects. the problem is that we cannot do that under spinlock
291 	 * or in irq handler context at all (since release functions are long and
292 	 * might sleep), so we will need to handle that part in workqueue context.
293 	 * To avoid handling kmalloc failure which compels us rolling back actions
294 	 * and move nodes hanged on the free list back to the interrupt wait list
295 	 * we always alloc the job of the WQ at the beginning.
296 	 */
297 	job = kmalloc(sizeof(*job), GFP_ATOMIC);
298 	if (!job)
299 		return;
300 
301 	spin_lock(&intr->wait_list_lock);
302 	list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, wait_list_node) {
303 		if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
304 				!pend->cq_kernel_addr) {
305 			if (pend->ts_reg_info.buf) {
306 				if (!reg_node_handle_fail) {
307 					rc = handle_registration_node(hdev, pend,
308 							&ts_reg_free_list_head, intr->timestamp);
309 					if (rc)
310 						reg_node_handle_fail = true;
311 				}
312 			} else {
313 				/* Handle wait target value node */
314 				pend->fence.timestamp = intr->timestamp;
315 				complete_all(&pend->fence.completion);
316 			}
317 		}
318 	}
319 	spin_unlock(&intr->wait_list_lock);
320 
321 	if (ts_reg_free_list_head) {
322 		INIT_WORK(&job->free_obj, hl_ts_free_objects);
323 		job->free_obj_head = ts_reg_free_list_head;
324 		job->hdev = hdev;
325 		queue_work(hdev->ts_free_obj_wq, &job->free_obj);
326 	} else {
327 		kfree(job);
328 	}
329 }
330 
handle_tpc_interrupt(struct hl_device * hdev)331 static void handle_tpc_interrupt(struct hl_device *hdev)
332 {
333 	u64 event_mask;
334 	u32 flags;
335 
336 	event_mask = HL_NOTIFIER_EVENT_TPC_ASSERT |
337 		HL_NOTIFIER_EVENT_USER_ENGINE_ERR |
338 		HL_NOTIFIER_EVENT_DEVICE_RESET;
339 
340 	flags = HL_DRV_RESET_DELAY;
341 
342 	dev_err_ratelimited(hdev->dev, "Received TPC assert\n");
343 	hl_device_cond_reset(hdev, flags, event_mask);
344 }
345 
handle_unexpected_user_interrupt(struct hl_device * hdev)346 static void handle_unexpected_user_interrupt(struct hl_device *hdev)
347 {
348 	dev_err_ratelimited(hdev->dev, "Received unexpected user error interrupt\n");
349 }
350 
351 /**
352  * hl_irq_handler_user_interrupt - irq handler for user interrupts
353  *
354  * @irq: irq number
355  * @arg: pointer to user interrupt structure
356  *
357  */
hl_irq_handler_user_interrupt(int irq,void * arg)358 irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
359 {
360 	struct hl_user_interrupt *user_int = arg;
361 
362 	user_int->timestamp = ktime_get();
363 
364 	return IRQ_WAKE_THREAD;
365 }
366 
367 /**
368  * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
369  * This function is invoked by threaded irq mechanism
370  *
371  * @irq: irq number
372  * @arg: pointer to user interrupt structure
373  *
374  */
hl_irq_user_interrupt_thread_handler(int irq,void * arg)375 irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
376 {
377 	struct hl_user_interrupt *user_int = arg;
378 	struct hl_device *hdev = user_int->hdev;
379 
380 	switch (user_int->type) {
381 	case HL_USR_INTERRUPT_CQ:
382 		handle_user_interrupt(hdev, &hdev->common_user_cq_interrupt);
383 
384 		/* Handle user cq interrupt registered on this specific irq */
385 		handle_user_interrupt(hdev, user_int);
386 		break;
387 	case HL_USR_INTERRUPT_DECODER:
388 		handle_user_interrupt(hdev, &hdev->common_decoder_interrupt);
389 
390 		/* Handle decoder interrupt registered on this specific irq */
391 		handle_user_interrupt(hdev, user_int);
392 		break;
393 	case HL_USR_INTERRUPT_TPC:
394 		handle_tpc_interrupt(hdev);
395 		break;
396 	case HL_USR_INTERRUPT_UNEXPECTED:
397 		handle_unexpected_user_interrupt(hdev);
398 		break;
399 	default:
400 		break;
401 	}
402 
403 	return IRQ_HANDLED;
404 }
405 
406 /**
407  * hl_irq_handler_eq - irq handler for event queue
408  *
409  * @irq: irq number
410  * @arg: pointer to event queue structure
411  *
412  */
hl_irq_handler_eq(int irq,void * arg)413 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
414 {
415 	struct hl_eq *eq = arg;
416 	struct hl_device *hdev = eq->hdev;
417 	struct hl_eq_entry *eq_entry;
418 	struct hl_eq_entry *eq_base;
419 	struct hl_eqe_work *handle_eqe_work;
420 	bool entry_ready;
421 	u32 cur_eqe, ctl;
422 	u16 cur_eqe_index, event_type;
423 
424 	eq_base = eq->kernel_address;
425 
426 	while (1) {
427 		cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
428 		entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
429 
430 		if (!entry_ready)
431 			break;
432 
433 		cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
434 		if ((hdev->event_queue.check_eqe_index) &&
435 				(((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) != cur_eqe_index)) {
436 			dev_err(hdev->dev,
437 				"EQE %#x in queue is ready but index does not match %d!=%d",
438 				cur_eqe,
439 				((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
440 				cur_eqe_index);
441 			break;
442 		}
443 
444 		eq->prev_eqe_index++;
445 
446 		eq_entry = &eq_base[eq->ci];
447 
448 		/*
449 		 * Make sure we read EQ entry contents after we've
450 		 * checked the ownership bit.
451 		 */
452 		dma_rmb();
453 
454 		if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
455 			ctl = le32_to_cpu(eq_entry->hdr.ctl);
456 			event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
457 			dev_warn(hdev->dev,
458 				"Device disabled but received an EQ event (%u)\n", event_type);
459 			goto skip_irq;
460 		}
461 
462 		handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
463 		if (handle_eqe_work) {
464 			INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
465 			handle_eqe_work->hdev = hdev;
466 
467 			memcpy(&handle_eqe_work->eq_entry, eq_entry,
468 					sizeof(*eq_entry));
469 
470 			queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
471 		}
472 skip_irq:
473 		/* Clear EQ entry ready bit */
474 		eq_entry->hdr.ctl =
475 			cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
476 							~EQ_CTL_READY_MASK);
477 
478 		eq->ci = hl_eq_inc_ptr(eq->ci);
479 
480 		hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
481 	}
482 
483 	return IRQ_HANDLED;
484 }
485 
486 /**
487  * hl_irq_handler_dec_abnrm - Decoder error interrupt handler
488  * @irq: IRQ number
489  * @arg: pointer to decoder structure.
490  */
hl_irq_handler_dec_abnrm(int irq,void * arg)491 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
492 {
493 	struct hl_dec *dec = arg;
494 
495 	schedule_work(&dec->abnrm_intr_work);
496 
497 	return IRQ_HANDLED;
498 }
499 
500 /**
501  * hl_cq_init - main initialization function for an cq object
502  *
503  * @hdev: pointer to device structure
504  * @q: pointer to cq structure
505  * @hw_queue_id: The H/W queue ID this completion queue belongs to
506  *               HL_INVALID_QUEUE if cq is not attached to any specific queue
507  *
508  * Allocate dma-able memory for the completion queue and initialize fields
509  * Returns 0 on success
510  */
hl_cq_init(struct hl_device * hdev,struct hl_cq * q,u32 hw_queue_id)511 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
512 {
513 	void *p;
514 
515 	p = hl_asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES, &q->bus_address,
516 					GFP_KERNEL | __GFP_ZERO);
517 	if (!p)
518 		return -ENOMEM;
519 
520 	q->hdev = hdev;
521 	q->kernel_address = p;
522 	q->hw_queue_id = hw_queue_id;
523 	q->ci = 0;
524 	q->pi = 0;
525 
526 	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
527 
528 	return 0;
529 }
530 
531 /**
532  * hl_cq_fini - destroy completion queue
533  *
534  * @hdev: pointer to device structure
535  * @q: pointer to cq structure
536  *
537  * Free the completion queue memory
538  */
hl_cq_fini(struct hl_device * hdev,struct hl_cq * q)539 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
540 {
541 	hl_asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES, q->kernel_address, q->bus_address);
542 }
543 
hl_cq_reset(struct hl_device * hdev,struct hl_cq * q)544 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
545 {
546 	q->ci = 0;
547 	q->pi = 0;
548 
549 	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
550 
551 	/*
552 	 * It's not enough to just reset the PI/CI because the H/W may have
553 	 * written valid completion entries before it was halted and therefore
554 	 * we need to clean the actual queues so we won't process old entries
555 	 * when the device is operational again
556 	 */
557 
558 	memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
559 }
560 
561 /**
562  * hl_eq_init - main initialization function for an event queue object
563  *
564  * @hdev: pointer to device structure
565  * @q: pointer to eq structure
566  *
567  * Allocate dma-able memory for the event queue and initialize fields
568  * Returns 0 on success
569  */
hl_eq_init(struct hl_device * hdev,struct hl_eq * q)570 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
571 {
572 	void *p;
573 
574 	p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_EQ_SIZE_IN_BYTES, &q->bus_address);
575 	if (!p)
576 		return -ENOMEM;
577 
578 	q->hdev = hdev;
579 	q->kernel_address = p;
580 	q->ci = 0;
581 	q->prev_eqe_index = 0;
582 
583 	return 0;
584 }
585 
586 /**
587  * hl_eq_fini - destroy event queue
588  *
589  * @hdev: pointer to device structure
590  * @q: pointer to eq structure
591  *
592  * Free the event queue memory
593  */
hl_eq_fini(struct hl_device * hdev,struct hl_eq * q)594 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
595 {
596 	flush_workqueue(hdev->eq_wq);
597 
598 	hl_cpu_accessible_dma_pool_free(hdev, HL_EQ_SIZE_IN_BYTES, q->kernel_address);
599 }
600 
hl_eq_reset(struct hl_device * hdev,struct hl_eq * q)601 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
602 {
603 	q->ci = 0;
604 	q->prev_eqe_index = 0;
605 
606 	/*
607 	 * It's not enough to just reset the PI/CI because the H/W may have
608 	 * written valid completion entries before it was halted and therefore
609 	 * we need to clean the actual queues so we won't process old entries
610 	 * when the device is operational again
611 	 */
612 
613 	memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
614 }
615