38 		       struct queue_properties *q,
42 				struct queue_properties *q)  in mqd_stride_v9()  argument
45 	    q->type == KFD_QUEUE_TYPE_COMPUTE)  in mqd_stride_v9()
46 		return ALIGN(q->ctl_stack_size, PAGE_SIZE) +  in mqd_stride_v9()
104 static void set_priority(struct v9_mqd *m, struct queue_properties *q)  in set_priority()  argument
106 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];  in set_priority()
107 	m->cp_hqd_queue_priority = q->priority;  in set_priority()
111 		struct queue_properties *q)  in allocate_mqd()  argument
132 	if (node->kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {  in allocate_mqd()
137 			(ALIGN(q->ctl_stack_size, PAGE_SIZE) +  in allocate_mqd()
160 			struct queue_properties *q)  in init_mqd()  argument
198 	if (q->format == KFD_QUEUE_FORMAT_AQL)  in init_mqd()
202 	if (q->tba_addr) {  in init_mqd()
207 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {  in init_mqd()
211 			lower_32_bits(q->ctx_save_restore_area_address);  in init_mqd()
213 			upper_32_bits(q->ctx_save_restore_area_address);  in init_mqd()
214 		m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;  in init_mqd()
215 		m->cp_hqd_cntl_stack_size = q->ctl_stack_size;  in init_mqd()
216 		m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;  in init_mqd()
217 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;  in init_mqd()
223 	update_mqd(mm, m, q, NULL);  in init_mqd()
239 			struct queue_properties *q,  in update_mqd()  argument
247 	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;  in update_mqd()
250 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);  in update_mqd()
251 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);  in update_mqd()
253 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);  in update_mqd()
254 	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);  in update_mqd()
255 	m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);  in update_mqd()
256 	m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);  in update_mqd()
259 		q->doorbell_off <<  in update_mqd()
279 	m->cp_hqd_eop_control = q->eop_ring_buffer_size ?  in update_mqd()
280 		min(0xA, order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0;  in update_mqd()
283 			lower_32_bits(q->eop_ring_buffer_address >> 8);  in update_mqd()
285 			upper_32_bits(q->eop_ring_buffer_address >> 8);  in update_mqd()
289 	m->cp_hqd_vmid = q->vmid;  in update_mqd()
291 	if (q->format == KFD_QUEUE_FORMAT_AQL) {  in update_mqd()
299 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)  in update_mqd()
304 	set_priority(m, q);  in update_mqd()
306 	q->is_active = QUEUE_IS_ACTIVE(*q);  in update_mqd()
318 			  struct queue_properties *q,  in get_wave_state()  argument
406 			struct queue_properties *q)  in init_mqd_hiq()  argument
410 	init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);  in init_mqd_hiq()
439 		struct queue_properties *q)  in init_mqd_sdma()  argument
451 	mm->update_mqd(mm, m, q, NULL);  in init_mqd_sdma()
457 			struct queue_properties *q,  in update_mqd_sdma()  argument
463 	m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)  in update_mqd_sdma()
465 		q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |  in update_mqd_sdma()
469 	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);  in update_mqd_sdma()
470 	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);  in update_mqd_sdma()
471 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);  in update_mqd_sdma()
472 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);  in update_mqd_sdma()
474 		q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;  in update_mqd_sdma()
476 	m->sdma_engine_id = q->sdma_engine_id;  in update_mqd_sdma()
477 	m->sdma_queue_id = q->sdma_queue_id;  in update_mqd_sdma()
480 	q->is_active = QUEUE_IS_ACTIVE(*q);  in update_mqd_sdma()
521 			struct queue_properties *q)  in init_mqd_hiq_v9_4_3()  argument
533 		init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);  in init_mqd_hiq_v9_4_3()
614 			struct queue_properties *q)  in init_mqd_v9_4_3()  argument
621 	uint64_t offset = mm->mqd_stride(mm, q);  in init_mqd_v9_4_3()
628 		init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);  in init_mqd_v9_4_3()
637 		    q->ctx_save_restore_area_address) {  in init_mqd_v9_4_3()
639 				q->ctx_save_restore_area_address +  in init_mqd_v9_4_3()
640 				(xcc * q->ctx_save_restore_area_size);  in init_mqd_v9_4_3()
648 		if (q->format == KFD_QUEUE_FORMAT_AQL) {  in init_mqd_v9_4_3()
667 			m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;  in init_mqd_v9_4_3()
679 		      struct queue_properties *q, struct mqd_update_info *minfo)  in update_mqd_v9_4_3()  argument
683 	uint64_t size = mm->mqd_stride(mm, q);  in update_mqd_v9_4_3()
687 		update_mqd(mm, m, q, minfo);  in update_mqd_v9_4_3()
691 		if (q->format == KFD_QUEUE_FORMAT_AQL) {  in update_mqd_v9_4_3()
706 			m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;  in update_mqd_v9_4_3()
767 				 struct queue_properties *q,  in get_wave_state_v9_4_3()  argument
775 	uint64_t mqd_stride_size = mm->mqd_stride(mm, q);  in get_wave_state_v9_4_3()
781 					q->ctx_save_restore_area_size * xcc);  in get_wave_state_v9_4_3()
783 		err = get_wave_state(mm, xcc_mqd, q, xcc_ctl_stack,  in get_wave_state_v9_4_3()