39 static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
41 static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
43 static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
48 static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
71 static void _cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,  in _cdx_mcdi_remove_cmd()  argument
81 		++mcdi->outstanding_cleanups;  in _cdx_mcdi_remove_cmd()
86 static void cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_remove_cmd()  argument
91 	_cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_remove_cmd()
94 	if (list_empty(&mcdi->cmd_list))  in cdx_mcdi_remove_cmd()
95 		wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_remove_cmd()
108 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_init()  local
111 	cdx->mcdi = kzalloc(sizeof(*cdx->mcdi), GFP_KERNEL);  in cdx_mcdi_init()
112 	if (!cdx->mcdi)  in cdx_mcdi_init()
115 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_init()
116 	mcdi->cdx = cdx;  in cdx_mcdi_init()
118 	mcdi->workqueue = alloc_ordered_workqueue("mcdi_wq", 0);  in cdx_mcdi_init()
119 	if (!mcdi->workqueue)  in cdx_mcdi_init()
121 	mutex_init(&mcdi->iface_lock);  in cdx_mcdi_init()
122 	mcdi->mode = MCDI_MODE_EVENTS;  in cdx_mcdi_init()
123 	INIT_LIST_HEAD(&mcdi->cmd_list);  in cdx_mcdi_init()
124 	init_waitqueue_head(&mcdi->cmd_complete_wq);  in cdx_mcdi_init()
126 	mcdi->new_epoch = true;  in cdx_mcdi_init()
130 	kfree(cdx->mcdi);  in cdx_mcdi_init()
131 	cdx->mcdi = NULL;  in cdx_mcdi_init()
138 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_finish()  local
140 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_finish()
141 	if (!mcdi)  in cdx_mcdi_finish()
146 	destroy_workqueue(mcdi->workqueue);  in cdx_mcdi_finish()
147 	kfree(cdx->mcdi);  in cdx_mcdi_finish()
148 	cdx->mcdi = NULL;  in cdx_mcdi_finish()
151 static bool cdx_mcdi_flushed(struct cdx_mcdi_iface *mcdi, bool ignore_cleanups)  in cdx_mcdi_flushed()  argument
155 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_flushed()
156 	flushed = list_empty(&mcdi->cmd_list) &&  in cdx_mcdi_flushed()
157 		  (ignore_cleanups || !mcdi->outstanding_cleanups);  in cdx_mcdi_flushed()
158 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_flushed()
165 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_wait_for_cleanup()  local
167 	if (!mcdi)  in cdx_mcdi_wait_for_cleanup()
170 	wait_event(mcdi->cmd_complete_wq,  in cdx_mcdi_wait_for_cleanup()
171 		   cdx_mcdi_flushed(mcdi, false));  in cdx_mcdi_wait_for_cleanup()
177 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_wait_for_quiescence()  local
181 	if (!mcdi)  in cdx_mcdi_wait_for_quiescence()
184 	flush_workqueue(mcdi->workqueue);  in cdx_mcdi_wait_for_quiescence()
186 	add_wait_queue(&mcdi->cmd_complete_wq, &wait);  in cdx_mcdi_wait_for_quiescence()
188 	while (!cdx_mcdi_flushed(mcdi, true)) {  in cdx_mcdi_wait_for_quiescence()
195 	remove_wait_queue(&mcdi->cmd_complete_wq, &wait);  in cdx_mcdi_wait_for_quiescence()
225 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_send_request()  local
233 	if (!mcdi)  in cdx_mcdi_send_request()
236 	mcdi->prev_seq = cmd->seq;  in cdx_mcdi_send_request()
237 	mcdi->seq_held_by[cmd->seq] = cmd;  in cdx_mcdi_send_request()
238 	mcdi->db_held_by = cmd;  in cdx_mcdi_send_request()
241 	not_epoch = !mcdi->new_epoch;  in cdx_mcdi_send_request()
269 	mcdi->new_epoch = false;  in cdx_mcdi_send_request()
322 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_process_cleanup_list()  local
325 	if (!mcdi)  in cdx_mcdi_process_cleanup_list()
342 		mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_process_cleanup_list()
343 		CDX_WARN_ON_PARANOID(cleanups > mcdi->outstanding_cleanups);  in cdx_mcdi_process_cleanup_list()
344 		all_done = (mcdi->outstanding_cleanups -= cleanups) == 0;  in cdx_mcdi_process_cleanup_list()
345 		mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cleanup_list()
347 			wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_process_cleanup_list()
351 static void _cdx_mcdi_cancel_cmd(struct cdx_mcdi_iface *mcdi,  in _cdx_mcdi_cancel_cmd()  argument
357 	list_for_each_entry(cmd, &mcdi->cmd_list, list)  in _cdx_mcdi_cancel_cmd()
366 				cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in _cdx_mcdi_cancel_cmd()
381 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_cancel_cmd()  local
384 	if (!mcdi)  in cdx_mcdi_cancel_cmd()
387 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_cancel_cmd()
388 	cdx_mcdi_timeout_cmd(mcdi, cmd, &cleanup_list);  in cdx_mcdi_cancel_cmd()
389 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_cancel_cmd()
493 static bool cdx_mcdi_get_seq(struct cdx_mcdi_iface *mcdi, unsigned char *seq)  in cdx_mcdi_get_seq()  argument
495 	*seq = mcdi->prev_seq;  in cdx_mcdi_get_seq()
497 		*seq = (*seq + 1) % ARRAY_SIZE(mcdi->seq_held_by);  in cdx_mcdi_get_seq()
498 	} while (mcdi->seq_held_by[*seq] && *seq != mcdi->prev_seq);  in cdx_mcdi_get_seq()
499 	return !mcdi->seq_held_by[*seq];  in cdx_mcdi_get_seq()
506 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_rpc_async_internal()  local
509 	if (!mcdi) {  in cdx_mcdi_rpc_async_internal()
514 	if (mcdi->mode == MCDI_MODE_FAIL) {  in cdx_mcdi_rpc_async_internal()
519 	cmd->mcdi = mcdi;  in cdx_mcdi_rpc_async_internal()
527 	queue_work(mcdi->workqueue, &cmd->work);  in cdx_mcdi_rpc_async_internal()
531 static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_cmd_start_or_queue()  argument
534 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_cmd_start_or_queue()
537 	if (!mcdi->db_held_by &&  in cdx_mcdi_cmd_start_or_queue()
538 	    cdx_mcdi_get_seq(mcdi, &seq)) {  in cdx_mcdi_cmd_start_or_queue()
549 static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_start_or_queue()  argument
554 	list_for_each_entry_safe(cmd, tmp, &mcdi->cmd_list, list)  in cdx_mcdi_start_or_queue()
557 			cdx_mcdi_cmd_start_or_queue(mcdi, cmd);  in cdx_mcdi_start_or_queue()
562 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_process_cmd()  local
572 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_process_cmd()
573 	if (!mcdi)  in cdx_mcdi_process_cmd()
578 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
579 	cmd = mcdi->seq_held_by[respseq];  in cdx_mcdi_process_cmd()
583 			mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
588 		cdx_mcdi_complete_cmd(mcdi, cmd, outbuf, len, &cleanup_list);  in cdx_mcdi_process_cmd()
593 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
595 	cdx_mcdi_process_cleanup_list(mcdi->cdx, &cleanup_list);  in cdx_mcdi_process_cmd()
602 	struct cdx_mcdi_iface *mcdi = cmd->mcdi;  in cdx_mcdi_cmd_work()  local
604 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_cmd_work()
606 	cmd->handle = mcdi->prev_handle++;  in cdx_mcdi_cmd_work()
607 	list_add_tail(&cmd->list, &mcdi->cmd_list);  in cdx_mcdi_cmd_work()
608 	cdx_mcdi_cmd_start_or_queue(mcdi, cmd);  in cdx_mcdi_cmd_work()
610 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_cmd_work()
618 static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_complete_cmd()  argument
625 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_complete_cmd()
686 	if (mcdi->db_held_by == cmd)  in cdx_mcdi_complete_cmd()
687 		mcdi->db_held_by = NULL;  in cdx_mcdi_complete_cmd()
699 		cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_complete_cmd()
704 	mcdi->seq_held_by[cmd->seq] = NULL;  in cdx_mcdi_complete_cmd()
706 	cdx_mcdi_start_or_queue(mcdi, rc != MC_CMD_ERR_QUEUE_FULL);  in cdx_mcdi_complete_cmd()
709 	wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_complete_cmd()
716 static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_timeout_cmd()  argument
720 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_timeout_cmd()
727 	cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_timeout_cmd()
821 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_mode_fail()  local
823 	if (!mcdi)  in cdx_mcdi_mode_fail()
826 	mcdi->mode = MCDI_MODE_FAIL;  in cdx_mcdi_mode_fail()
828 	while (!list_empty(&mcdi->cmd_list)) {  in cdx_mcdi_mode_fail()
831 		cmd = list_first_entry(&mcdi->cmd_list, struct cdx_mcdi_cmd,  in cdx_mcdi_mode_fail()
833 		_cdx_mcdi_cancel_cmd(mcdi, cdx_mcdi_cmd_handle(cmd), cleanup_list);  in cdx_mcdi_mode_fail()