78 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \  argument
79 					     struct hinic_cmdqs, cmdq[0])
320 static void cmdq_set_db(struct hinic_cmdq *cmdq,  in cmdq_set_db()  argument
332 	writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));  in cmdq_set_db()
335 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,  in cmdq_sync_cmd_direct_resp()  argument
343 	struct hinic_wq *wq = cmdq->wq;  in cmdq_sync_cmd_direct_resp()
348 	spin_lock_bh(&cmdq->cmdq_lock);  in cmdq_sync_cmd_direct_resp()
353 		spin_unlock_bh(&cmdq->cmdq_lock);  in cmdq_sync_cmd_direct_resp()
359 	wrapped = cmdq->wrapped;  in cmdq_sync_cmd_direct_resp()
364 		cmdq->wrapped = !cmdq->wrapped;  in cmdq_sync_cmd_direct_resp()
368 	cmdq->errcode[curr_prod_idx] = &errcode;  in cmdq_sync_cmd_direct_resp()
371 	cmdq->done[curr_prod_idx] = &done;  in cmdq_sync_cmd_direct_resp()
380 	/* CMDQ WQE is not shadow, therefore wqe will be written to wq */  in cmdq_sync_cmd_direct_resp()
383 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);  in cmdq_sync_cmd_direct_resp()
385 	spin_unlock_bh(&cmdq->cmdq_lock);  in cmdq_sync_cmd_direct_resp()
389 		spin_lock_bh(&cmdq->cmdq_lock);  in cmdq_sync_cmd_direct_resp()
391 		if (cmdq->errcode[curr_prod_idx] == &errcode)  in cmdq_sync_cmd_direct_resp()
392 			cmdq->errcode[curr_prod_idx] = NULL;  in cmdq_sync_cmd_direct_resp()
394 		if (cmdq->done[curr_prod_idx] == &done)  in cmdq_sync_cmd_direct_resp()
395 			cmdq->done[curr_prod_idx] = NULL;  in cmdq_sync_cmd_direct_resp()
397 		spin_unlock_bh(&cmdq->cmdq_lock);  in cmdq_sync_cmd_direct_resp()
399 		hinic_dump_ceq_info(cmdq->hwdev);  in cmdq_sync_cmd_direct_resp()
417 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in,  in cmdq_set_arm_bit()  argument
422 	struct hinic_wq *wq = cmdq->wq;  in cmdq_set_arm_bit()
427 	spin_lock(&cmdq->cmdq_lock);  in cmdq_set_arm_bit()
432 		spin_unlock(&cmdq->cmdq_lock);  in cmdq_set_arm_bit()
438 	wrapped = cmdq->wrapped;  in cmdq_set_arm_bit()
443 		cmdq->wrapped = !cmdq->wrapped;  in cmdq_set_arm_bit()
454 	/* cmdq wqe is not shadow, therefore wqe will be written to wq */  in cmdq_set_arm_bit()
457 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);  in cmdq_set_arm_bit()
459 	spin_unlock(&cmdq->cmdq_lock);  in cmdq_set_arm_bit()
491 		dev_err(&pdev->dev, "Invalid CMDQ parameters\n");  in hinic_cmdq_direct_resp()
495 	return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],  in hinic_cmdq_direct_resp()
510 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];  in hinic_set_arm_bit()  local
519 	err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit));  in hinic_set_arm_bit()
528 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,  in clear_wqe_complete_bit()  argument
556  * cmdq_arm_ceq_handler - cmdq completion event handler for arm command
557  * @cmdq: the cmdq of the arm command
562 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq,  in cmdq_arm_ceq_handler()  argument
578 	clear_wqe_complete_bit(cmdq, wqe);  in cmdq_arm_ceq_handler()
580 	hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE);  in cmdq_arm_ceq_handler()
584 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,  in cmdq_update_errcode()  argument
587 	if (cmdq->errcode[prod_idx])  in cmdq_update_errcode()
588 		*cmdq->errcode[prod_idx] = errcode;  in cmdq_update_errcode()
592  * cmdq_sync_cmd_handler - cmdq completion event handler for sync command
593  * @cmdq: the cmdq of the command
597 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx,  in cmdq_sync_cmd_handler()  argument
602 	spin_lock(&cmdq->cmdq_lock);  in cmdq_sync_cmd_handler()
603 	cmdq_update_errcode(cmdq, prod_idx, errcode);  in cmdq_sync_cmd_handler()
607 	if (cmdq->done[prod_idx])  in cmdq_sync_cmd_handler()
608 		complete(cmdq->done[prod_idx]);  in cmdq_sync_cmd_handler()
609 	spin_unlock(&cmdq->cmdq_lock);  in cmdq_sync_cmd_handler()
612 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci,  in cmdq_cmd_ceq_handler()  argument
627 	cmdq_sync_cmd_handler(cmdq, ci, errcode);  in cmdq_cmd_ceq_handler()
629 	clear_wqe_complete_bit(cmdq, cmdq_wqe);  in cmdq_cmd_ceq_handler()
630 	hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE);  in cmdq_cmd_ceq_handler()
635  * cmdq_ceq_handler - cmdq completion event handler
643 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type];  in cmdq_ceq_handler()  local
651 	while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) {  in cmdq_ceq_handler()
662 			if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe))  in cmdq_ceq_handler()
667 			hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci);  in cmdq_ceq_handler()
671 			if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe))  in cmdq_ceq_handler()
682 			dev_err(&pdev->dev, "Failed to set arm for CMDQ\n");  in cmdq_ceq_handler()
687  * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq
688  * @cmdq_ctxt: cmdq ctxt to initialize
689  * @cmdq: the cmdq
693 				 struct hinic_cmdq *cmdq,  in cmdq_init_queue_ctxt()  argument
698 	struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq);  in cmdq_init_queue_ctxt()
699 	struct hinic_wq *wq = cmdq->wq;  in cmdq_init_queue_ctxt()
711 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED);  in cmdq_init_queue_ctxt()
726 	cmdq_ctxt->cmdq_type  = cmdq->cmdq_type;  in cmdq_init_queue_ctxt()
730  * init_cmdq - initialize cmdq
731  * @cmdq: the cmdq
732  * @wq: the wq attaced to the cmdq
733  * @q_type: the cmdq type of the cmdq
734  * @db_area: doorbell area for the cmdq
738 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq,  in init_cmdq()  argument
743 	cmdq->wq = wq;  in init_cmdq()
744 	cmdq->cmdq_type = q_type;  in init_cmdq()
745 	cmdq->wrapped = 1;  in init_cmdq()
747 	spin_lock_init(&cmdq->cmdq_lock);  in init_cmdq()
749 	cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth));  in init_cmdq()
750 	if (!cmdq->done)  in init_cmdq()
753 	cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode),  in init_cmdq()
755 	if (!cmdq->errcode) {  in init_cmdq()
760 	cmdq->db_base = db_area + CMDQ_DB_OFF;  in init_cmdq()
764 	vfree(cmdq->done);  in init_cmdq()
769  * free_cmdq - Free cmdq
770  * @cmdq: the cmdq to free
772 static void free_cmdq(struct hinic_cmdq *cmdq)  in free_cmdq()  argument
774 	vfree(cmdq->errcode);  in free_cmdq()
775 	vfree(cmdq->done);  in free_cmdq()
779  * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq
805 		cmdqs->cmdq[cmdq_type].hwdev = hwdev;  in init_cmdqs_ctxt()
806 		err = init_cmdq(&cmdqs->cmdq[cmdq_type],  in init_cmdqs_ctxt()
810 			dev_err(&pdev->dev, "Failed to initialize cmdq\n");  in init_cmdqs_ctxt()
815 				     &cmdqs->cmdq[cmdq_type],  in init_cmdqs_ctxt()
819 	/* Write the CMDQ ctxts */  in init_cmdqs_ctxt()
828 			dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n",  in init_cmdqs_ctxt()
842 		free_cmdq(&cmdqs->cmdq[type]);  in init_cmdqs_ctxt()
903 		dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n");  in hinic_init_cmdqs()
910 		dev_err(&pdev->dev, "Failed to write cmdq ctxt\n");  in hinic_init_cmdqs()
919 		dev_err(&hwif->pdev->dev, "Failed to set cmdq depth\n");  in hinic_init_cmdqs()
927 	free_cmdq(&cmdqs->cmdq[HINIC_CMDQ_SYNC]);  in hinic_init_cmdqs()
955 		free_cmdq(&cmdqs->cmdq[cmdq_type]);  in hinic_free_cmdqs()