31 static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)  in queue_free_slots()  argument
33 	int delta = (q->pi - queue_ci_get(&q->ci, queue_len));  in queue_free_slots()
44 	struct hl_hw_queue *q;  in hl_hw_queue_update_ci()  local
50 	q = &hdev->kernel_queues[0];  in hl_hw_queue_update_ci()
53 	if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)  in hl_hw_queue_update_ci()
61 	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {  in hl_hw_queue_update_ci()
62 		if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)  in hl_hw_queue_update_ci()
63 			atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);  in hl_hw_queue_update_ci()
83 void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,  in hl_hw_queue_submit_bd()  argument
88 	bd = q->kernel_address;  in hl_hw_queue_submit_bd()
89 	bd += hl_pi_2_offset(q->pi);  in hl_hw_queue_submit_bd()
94 	q->pi = hl_queue_inc_ptr(q->pi);  in hl_hw_queue_submit_bd()
95 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in hl_hw_queue_submit_bd()
117 				struct hl_hw_queue *q, int num_of_entries,  in ext_queue_sanity_checks()  argument
121 			&hdev->completion_queue[q->cq_id].free_slots_cnt;  in ext_queue_sanity_checks()
125 	free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);  in ext_queue_sanity_checks()
129 			q->hw_queue_id, num_of_entries);  in ext_queue_sanity_checks()
143 				num_of_entries, q->hw_queue_id);  in ext_queue_sanity_checks()
166 					struct hl_hw_queue *q,  in int_queue_sanity_checks()  argument
171 	if (num_of_entries > q->int_queue_len) {  in int_queue_sanity_checks()
174 			q->hw_queue_id, num_of_entries);  in int_queue_sanity_checks()
179 	free_slots_cnt = queue_free_slots(q, q->int_queue_len);  in int_queue_sanity_checks()
183 			q->hw_queue_id, num_of_entries);  in int_queue_sanity_checks()
200 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,  in hw_queue_sanity_checks()  argument
206 	free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);  in hw_queue_sanity_checks()
210 			q->hw_queue_id, num_of_entries);  in hw_queue_sanity_checks()
231 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_send_cb_no_cmpl()  local
246 	if (q->queue_type != QUEUE_TYPE_HW) {  in hl_hw_queue_send_cb_no_cmpl()
247 		rc = ext_queue_sanity_checks(hdev, q, 1, false);  in hl_hw_queue_send_cb_no_cmpl()
252 	hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);  in hl_hw_queue_send_cb_no_cmpl()
271 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in ext_queue_schedule_job()  local
284 	ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);  in ext_queue_schedule_job()
295 			((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)  in ext_queue_schedule_job()
307 	cq = &hdev->completion_queue[q->cq_id];  in ext_queue_schedule_job()
314 						q->msi_vec,  in ext_queue_schedule_job()
317 	q->shadow_queue[hl_pi_2_offset(q->pi)] = job;  in ext_queue_schedule_job()
322 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in ext_queue_schedule_job()
336 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in int_queue_schedule_job()  local
351 	pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd);  in int_queue_schedule_job()
353 	q->pi++;  in int_queue_schedule_job()
354 	q->pi &= ((q->int_queue_len << 1) - 1);  in int_queue_schedule_job()
358 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in int_queue_schedule_job()
372 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in hw_queue_schedule_job()  local
384 		((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK);  in hw_queue_schedule_job()
401 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in hw_queue_schedule_job()
632 	struct hl_hw_queue *q;  in hl_hw_queue_schedule_cs()  local
652 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
653 	for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {  in hl_hw_queue_schedule_cs()
655 			switch (q->queue_type) {  in hl_hw_queue_schedule_cs()
657 				rc = ext_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
663 				rc = int_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
667 				rc = hw_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
672 					q->queue_type);  in hl_hw_queue_schedule_cs()
684 			if (q->queue_type == QUEUE_TYPE_EXT)  in hl_hw_queue_schedule_cs()
783 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
784 	for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {  in hl_hw_queue_schedule_cs()
785 		if ((q->queue_type == QUEUE_TYPE_EXT) &&  in hl_hw_queue_schedule_cs()
808 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_inc_ci_kernel()  local
810 	atomic_inc(&q->ci);  in hl_hw_queue_inc_ci_kernel()
813 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in ext_and_cpu_queue_init()  argument
820 		p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address);  in ext_and_cpu_queue_init()
822 		p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in ext_and_cpu_queue_init()
827 	q->kernel_address = p;  in ext_and_cpu_queue_init()
829 	q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, sizeof(struct hl_cs_job *), GFP_KERNEL);  in ext_and_cpu_queue_init()
830 	if (!q->shadow_queue) {  in ext_and_cpu_queue_init()
833 			q->hw_queue_id);  in ext_and_cpu_queue_init()
839 	atomic_set(&q->ci, 0);  in ext_and_cpu_queue_init()
840 	q->pi = 0;  in ext_and_cpu_queue_init()
846 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in ext_and_cpu_queue_init()
848 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in ext_and_cpu_queue_init()
849 						q->bus_address);  in ext_and_cpu_queue_init()
854 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in int_queue_init()  argument
858 	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,  in int_queue_init()
859 					&q->bus_address, &q->int_queue_len);  in int_queue_init()
863 			q->hw_queue_id);  in int_queue_init()
867 	q->kernel_address = p;  in int_queue_init()
868 	q->pi = 0;  in int_queue_init()
869 	atomic_set(&q->ci, 0);  in int_queue_init()
874 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in cpu_queue_init()  argument
876 	return ext_and_cpu_queue_init(hdev, q, true);  in cpu_queue_init()
879 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in ext_queue_init()  argument
881 	return ext_and_cpu_queue_init(hdev, q, false);  in ext_queue_init()
884 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in hw_queue_init()  argument
888 	p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in hw_queue_init()
893 	q->kernel_address = p;  in hw_queue_init()
896 	atomic_set(&q->ci, 0);  in hw_queue_init()
897 	q->pi = 0;  in hw_queue_init()
985 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in queue_init()  argument
990 	q->hw_queue_id = hw_queue_id;  in queue_init()
992 	switch (q->queue_type) {  in queue_init()
994 		rc = ext_queue_init(hdev, q);  in queue_init()
997 		rc = int_queue_init(hdev, q);  in queue_init()
1000 		rc = cpu_queue_init(hdev, q);  in queue_init()
1003 		rc = hw_queue_init(hdev, q);  in queue_init()
1006 		q->valid = 0;  in queue_init()
1010 			q->queue_type);  in queue_init()
1015 	sync_stream_queue_init(hdev, q->hw_queue_id);  in queue_init()
1020 	q->valid = 1;  in queue_init()
1033 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)  in queue_fini()  argument
1035 	if (!q->valid)  in queue_fini()
1056 	if (q->queue_type == QUEUE_TYPE_INT)  in queue_fini()
1059 	kfree(q->shadow_queue);  in queue_fini()
1061 	if (q->queue_type == QUEUE_TYPE_CPU)  in queue_fini()
1062 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in queue_fini()
1064 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in queue_fini()
1065 						q->bus_address);  in queue_fini()
1071 	struct hl_hw_queue *q;  in hl_hw_queues_create()  local
1083 	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;  in hl_hw_queues_create()
1084 			i < asic->max_queues ; i++, q_ready_cnt++, q++) {  in hl_hw_queues_create()
1086 		q->queue_type = asic->hw_queues_props[i].type;  in hl_hw_queues_create()
1087 		q->supports_sync_stream =  in hl_hw_queues_create()
1089 		q->collective_mode = asic->hw_queues_props[i].collective_mode;  in hl_hw_queues_create()
1090 		rc = queue_init(hdev, q, i);  in hl_hw_queues_create()
1101 	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)  in hl_hw_queues_create()
1102 		queue_fini(hdev, q);  in hl_hw_queues_create()
1111 	struct hl_hw_queue *q;  in hl_hw_queues_destroy()  local
1115 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)  in hl_hw_queues_destroy()
1116 		queue_fini(hdev, q);  in hl_hw_queues_destroy()
1123 	struct hl_hw_queue *q;  in hl_hw_queue_reset()  local
1127 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {  in hl_hw_queue_reset()
1128 		if ((!q->valid) ||  in hl_hw_queue_reset()
1129 			((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))  in hl_hw_queue_reset()
1131 		q->pi = 0;  in hl_hw_queue_reset()
1132 		atomic_set(&q->ci, 0);  in hl_hw_queue_reset()
1134 		if (q->supports_sync_stream)  in hl_hw_queue_reset()
1135 			sync_stream_queue_reset(hdev, q->hw_queue_id);  in hl_hw_queue_reset()