275 #define GET_RX_CHANNEL_FLAG(channel, param)				     \  argument
277 		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \
278 		channel->info_word ?					     \
279 			le32_to_cpu(channel->info_word->rx.param) :	     \
280 			channel->info->rx.param;			     \
283 #define GET_RX_CHANNEL_INFO(channel, param)				      \  argument
285 		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \
286 		le32_to_cpu(channel->info_word ?			      \
287 			channel->info_word->rx.param :			      \
288 			channel->info->rx.param);			      \
291 #define SET_RX_CHANNEL_FLAG(channel, param, value)			     \  argument
293 		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \
294 		if (channel->info_word)					     \
295 			channel->info_word->rx.param = cpu_to_le32(value);   \
297 			channel->info->rx.param = value;		     \
300 #define SET_RX_CHANNEL_INFO(channel, param, value)			      \  argument
302 		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \
303 		if (channel->info_word)					      \
304 			channel->info_word->rx.param = cpu_to_le32(value);    \
306 			channel->info->rx.param = cpu_to_le32(value);	      \
309 #define GET_TX_CHANNEL_FLAG(channel, param)				     \  argument
311 		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \
312 		channel->info_word ?					     \
313 			le32_to_cpu(channel->info_word->tx.param) :          \
314 			channel->info->tx.param;			     \
317 #define GET_TX_CHANNEL_INFO(channel, param)				      \  argument
319 		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \
320 		le32_to_cpu(channel->info_word ?			      \
321 			channel->info_word->tx.param :			      \
322 			channel->info->tx.param);			      \
325 #define SET_TX_CHANNEL_FLAG(channel, param, value)			     \  argument
327 		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \
328 		if (channel->info_word)					     \
329 			channel->info_word->tx.param = cpu_to_le32(value);   \
331 			channel->info->tx.param = value;		     \
334 #define SET_TX_CHANNEL_INFO(channel, param, value)			      \  argument
336 		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \
337 		if (channel->info_word)					      \
338 			channel->info_word->tx.param = cpu_to_le32(value);   \
340 			channel->info->tx.param = cpu_to_le32(value);	      \
370 static void qcom_smd_signal_channel(struct qcom_smd_channel *channel)  in qcom_smd_signal_channel()  argument
372 	struct qcom_smd_edge *edge = channel->edge;  in qcom_smd_signal_channel()
390 static void qcom_smd_channel_reset(struct qcom_smd_channel *channel)  in qcom_smd_channel_reset()  argument
392 	SET_TX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED);  in qcom_smd_channel_reset()
393 	SET_TX_CHANNEL_FLAG(channel, fDSR, 0);  in qcom_smd_channel_reset()
394 	SET_TX_CHANNEL_FLAG(channel, fCTS, 0);  in qcom_smd_channel_reset()
395 	SET_TX_CHANNEL_FLAG(channel, fCD, 0);  in qcom_smd_channel_reset()
396 	SET_TX_CHANNEL_FLAG(channel, fRI, 0);  in qcom_smd_channel_reset()
397 	SET_TX_CHANNEL_FLAG(channel, fHEAD, 0);  in qcom_smd_channel_reset()
398 	SET_TX_CHANNEL_FLAG(channel, fTAIL, 0);  in qcom_smd_channel_reset()
399 	SET_TX_CHANNEL_FLAG(channel, fSTATE, 1);  in qcom_smd_channel_reset()
400 	SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1);  in qcom_smd_channel_reset()
401 	SET_TX_CHANNEL_INFO(channel, head, 0);  in qcom_smd_channel_reset()
402 	SET_RX_CHANNEL_INFO(channel, tail, 0);  in qcom_smd_channel_reset()
404 	qcom_smd_signal_channel(channel);  in qcom_smd_channel_reset()
406 	channel->state = SMD_CHANNEL_CLOSED;  in qcom_smd_channel_reset()
407 	channel->pkt_size = 0;  in qcom_smd_channel_reset()
413 static void qcom_smd_channel_set_callback(struct qcom_smd_channel *channel,  in qcom_smd_channel_set_callback()  argument
416 	struct rpmsg_endpoint *ept = &channel->qsept->ept;  in qcom_smd_channel_set_callback()
419 	spin_lock_irqsave(&channel->recv_lock, flags);  in qcom_smd_channel_set_callback()
421 	spin_unlock_irqrestore(&channel->recv_lock, flags);  in qcom_smd_channel_set_callback()
427 static size_t qcom_smd_channel_get_rx_avail(struct qcom_smd_channel *channel)  in qcom_smd_channel_get_rx_avail()  argument
432 	head = GET_RX_CHANNEL_INFO(channel, head);  in qcom_smd_channel_get_rx_avail()
433 	tail = GET_RX_CHANNEL_INFO(channel, tail);  in qcom_smd_channel_get_rx_avail()
435 	return (head - tail) & (channel->fifo_size - 1);  in qcom_smd_channel_get_rx_avail()
441 static void qcom_smd_channel_set_state(struct qcom_smd_channel *channel,  in qcom_smd_channel_set_state()  argument
444 	struct qcom_smd_edge *edge = channel->edge;  in qcom_smd_channel_set_state()
447 	if (channel->state == state)  in qcom_smd_channel_set_state()
450 	dev_dbg(&edge->dev, "set_state(%s, %d)\n", channel->name, state);  in qcom_smd_channel_set_state()
452 	SET_TX_CHANNEL_FLAG(channel, fDSR, is_open);  in qcom_smd_channel_set_state()
453 	SET_TX_CHANNEL_FLAG(channel, fCTS, is_open);  in qcom_smd_channel_set_state()
454 	SET_TX_CHANNEL_FLAG(channel, fCD, is_open);  in qcom_smd_channel_set_state()
456 	SET_TX_CHANNEL_INFO(channel, state, state);  in qcom_smd_channel_set_state()
457 	SET_TX_CHANNEL_FLAG(channel, fSTATE, 1);  in qcom_smd_channel_set_state()
459 	channel->state = state;  in qcom_smd_channel_set_state()
460 	qcom_smd_signal_channel(channel);  in qcom_smd_channel_set_state()
497 static size_t qcom_smd_channel_peek(struct qcom_smd_channel *channel,  in qcom_smd_channel_peek()  argument
504 	word_aligned = channel->info_word;  in qcom_smd_channel_peek()
505 	tail = GET_RX_CHANNEL_INFO(channel, tail);  in qcom_smd_channel_peek()
507 	len = min_t(size_t, count, channel->fifo_size - tail);  in qcom_smd_channel_peek()
510 				   channel->rx_fifo + tail,  in qcom_smd_channel_peek()
517 				   channel->rx_fifo,  in qcom_smd_channel_peek()
528 static void qcom_smd_channel_advance(struct qcom_smd_channel *channel,  in qcom_smd_channel_advance()  argument
533 	tail = GET_RX_CHANNEL_INFO(channel, tail);  in qcom_smd_channel_advance()
535 	tail &= (channel->fifo_size - 1);  in qcom_smd_channel_advance()
536 	SET_RX_CHANNEL_INFO(channel, tail, tail);  in qcom_smd_channel_advance()
542 static int qcom_smd_channel_recv_single(struct qcom_smd_channel *channel)  in qcom_smd_channel_recv_single()  argument
544 	struct rpmsg_endpoint *ept = &channel->qsept->ept;  in qcom_smd_channel_recv_single()
550 	tail = GET_RX_CHANNEL_INFO(channel, tail);  in qcom_smd_channel_recv_single()
553 	if (tail + channel->pkt_size >= channel->fifo_size) {  in qcom_smd_channel_recv_single()
554 		ptr = channel->bounce_buffer;  in qcom_smd_channel_recv_single()
555 		len = qcom_smd_channel_peek(channel, ptr, channel->pkt_size);  in qcom_smd_channel_recv_single()
557 		ptr = channel->rx_fifo + tail;  in qcom_smd_channel_recv_single()
558 		len = channel->pkt_size;  in qcom_smd_channel_recv_single()
566 	qcom_smd_channel_advance(channel, len);  in qcom_smd_channel_recv_single()
568 	channel->pkt_size = 0;  in qcom_smd_channel_recv_single()
576 static bool qcom_smd_channel_intr(struct qcom_smd_channel *channel)  in qcom_smd_channel_intr()  argument
585 	remote_state = GET_RX_CHANNEL_INFO(channel, state);  in qcom_smd_channel_intr()
586 	if (remote_state != channel->remote_state) {  in qcom_smd_channel_intr()
587 		channel->remote_state = remote_state;  in qcom_smd_channel_intr()
590 		wake_up_interruptible_all(&channel->state_change_event);  in qcom_smd_channel_intr()
593 	SET_RX_CHANNEL_FLAG(channel, fSTATE, 0);  in qcom_smd_channel_intr()
596 	if (!GET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR))  in qcom_smd_channel_intr()
597 		wake_up_interruptible_all(&channel->fblockread_event);  in qcom_smd_channel_intr()
600 	if (channel->state != SMD_CHANNEL_OPENED)  in qcom_smd_channel_intr()
604 	SET_RX_CHANNEL_FLAG(channel, fHEAD, 0);  in qcom_smd_channel_intr()
608 		avail = qcom_smd_channel_get_rx_avail(channel);  in qcom_smd_channel_intr()
610 		if (!channel->pkt_size && avail >= SMD_PACKET_HEADER_LEN) {  in qcom_smd_channel_intr()
611 			qcom_smd_channel_peek(channel, &pktlen, sizeof(pktlen));  in qcom_smd_channel_intr()
612 			qcom_smd_channel_advance(channel, SMD_PACKET_HEADER_LEN);  in qcom_smd_channel_intr()
613 			channel->pkt_size = le32_to_cpu(pktlen);  in qcom_smd_channel_intr()
614 		} else if (channel->pkt_size && avail >= channel->pkt_size) {  in qcom_smd_channel_intr()
615 			ret = qcom_smd_channel_recv_single(channel);  in qcom_smd_channel_intr()
624 	SET_RX_CHANNEL_FLAG(channel, fTAIL, 1);  in qcom_smd_channel_intr()
627 	if (!GET_RX_CHANNEL_FLAG(channel, fBLOCKREADINTR)) {  in qcom_smd_channel_intr()
631 		qcom_smd_signal_channel(channel);  in qcom_smd_channel_intr()
645 	struct qcom_smd_channel *channel;  in qcom_smd_edge_intr()  local
654 	list_for_each_entry(channel, &edge->channels, list) {  in qcom_smd_edge_intr()
655 		spin_lock(&channel->recv_lock);  in qcom_smd_edge_intr()
656 		kick_state |= qcom_smd_channel_intr(channel);  in qcom_smd_edge_intr()
657 		spin_unlock(&channel->recv_lock);  in qcom_smd_edge_intr()
683 static size_t qcom_smd_get_tx_avail(struct qcom_smd_channel *channel)  in qcom_smd_get_tx_avail()  argument
687 	unsigned mask = channel->fifo_size - 1;  in qcom_smd_get_tx_avail()
689 	head = GET_TX_CHANNEL_INFO(channel, head);  in qcom_smd_get_tx_avail()
690 	tail = GET_TX_CHANNEL_INFO(channel, tail);  in qcom_smd_get_tx_avail()
698 static int qcom_smd_write_fifo(struct qcom_smd_channel *channel,  in qcom_smd_write_fifo()  argument
706 	word_aligned = channel->info_word;  in qcom_smd_write_fifo()
707 	head = GET_TX_CHANNEL_INFO(channel, head);  in qcom_smd_write_fifo()
709 	len = min_t(size_t, count, channel->fifo_size - head);  in qcom_smd_write_fifo()
711 		smd_copy_to_fifo(channel->tx_fifo + head,  in qcom_smd_write_fifo()
718 		smd_copy_to_fifo(channel->tx_fifo,  in qcom_smd_write_fifo()
725 	head &= (channel->fifo_size - 1);  in qcom_smd_write_fifo()
726 	SET_TX_CHANNEL_INFO(channel, head, head);  in qcom_smd_write_fifo()
743 static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data,  in __qcom_smd_send()  argument
752 	if (channel->info_word && len % 4)  in __qcom_smd_send()
756 	if (tlen >= channel->fifo_size)  in __qcom_smd_send()
763 	spin_lock_irqsave(&channel->tx_lock, flags);  in __qcom_smd_send()
765 	while (qcom_smd_get_tx_avail(channel) < tlen &&  in __qcom_smd_send()
766 	       channel->state == SMD_CHANNEL_OPENED) {  in __qcom_smd_send()
772 		SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 0);  in __qcom_smd_send()
775 		spin_unlock_irqrestore(&channel->tx_lock, flags);  in __qcom_smd_send()
777 		ret = wait_event_interruptible(channel->fblockread_event,  in __qcom_smd_send()
778 				       qcom_smd_get_tx_avail(channel) >= tlen ||  in __qcom_smd_send()
779 				       channel->state != SMD_CHANNEL_OPENED);  in __qcom_smd_send()
783 		spin_lock_irqsave(&channel->tx_lock, flags);  in __qcom_smd_send()
785 		SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1);  in __qcom_smd_send()
789 	if (channel->state != SMD_CHANNEL_OPENED) {  in __qcom_smd_send()
794 	SET_TX_CHANNEL_FLAG(channel, fTAIL, 0);  in __qcom_smd_send()
796 	qcom_smd_write_fifo(channel, hdr, sizeof(hdr));  in __qcom_smd_send()
797 	qcom_smd_write_fifo(channel, data, len);  in __qcom_smd_send()
799 	SET_TX_CHANNEL_FLAG(channel, fHEAD, 1);  in __qcom_smd_send()
804 	qcom_smd_signal_channel(channel);  in __qcom_smd_send()
807 	spin_unlock_irqrestore(&channel->tx_lock, flags);  in __qcom_smd_send()
815 static int qcom_smd_channel_open(struct qcom_smd_channel *channel,  in qcom_smd_channel_open()  argument
818 	struct qcom_smd_edge *edge = channel->edge;  in qcom_smd_channel_open()
825 	bb_size = min(channel->fifo_size, SZ_4K);  in qcom_smd_channel_open()
826 	channel->bounce_buffer = kmalloc(bb_size, GFP_KERNEL);  in qcom_smd_channel_open()
827 	if (!channel->bounce_buffer)  in qcom_smd_channel_open()
830 	qcom_smd_channel_set_callback(channel, cb);  in qcom_smd_channel_open()
831 	qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENING);  in qcom_smd_channel_open()
834 	ret = wait_event_interruptible_timeout(channel->state_change_event,  in qcom_smd_channel_open()
835 			channel->remote_state == SMD_CHANNEL_OPENING ||  in qcom_smd_channel_open()
836 			channel->remote_state == SMD_CHANNEL_OPENED,  in qcom_smd_channel_open()
843 	qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENED);  in qcom_smd_channel_open()
846 	ret = wait_event_interruptible_timeout(channel->state_change_event,  in qcom_smd_channel_open()
847 			channel->remote_state == SMD_CHANNEL_OPENED,  in qcom_smd_channel_open()
857 	qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED);  in qcom_smd_channel_open()
864 static void qcom_smd_channel_close(struct qcom_smd_channel *channel)  in qcom_smd_channel_close()  argument
866 	qcom_smd_channel_set_callback(channel, NULL);  in qcom_smd_channel_close()
868 	kfree(channel->bounce_buffer);  in qcom_smd_channel_close()
869 	channel->bounce_buffer = NULL;  in qcom_smd_channel_close()
871 	qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED);  in qcom_smd_channel_close()
872 	qcom_smd_channel_reset(channel);  in qcom_smd_channel_close()
878 	struct qcom_smd_channel *channel;  in qcom_smd_find_channel()  local
883 	list_for_each_entry(channel, &edge->channels, list) {  in qcom_smd_find_channel()
884 		if (!strcmp(channel->name, name)) {  in qcom_smd_find_channel()
885 			ret = channel;  in qcom_smd_find_channel()
906 	struct qcom_smd_channel *channel;  in qcom_smd_create_ept()  local
915 			(channel = qcom_smd_find_channel(edge, name)) != NULL,  in qcom_smd_create_ept()
920 	if (channel->state != SMD_CHANNEL_CLOSED) {  in qcom_smd_create_ept()
921 		dev_err(&rpdev->dev, "channel %s is busy\n", channel->name);  in qcom_smd_create_ept()
938 	channel->qsept = qsept;  in qcom_smd_create_ept()
939 	qsept->qsch = channel;  in qcom_smd_create_ept()
941 	ret = qcom_smd_channel_open(channel, cb);  in qcom_smd_create_ept()
948 	channel->qsept = NULL;  in qcom_smd_create_ept()
995 	struct qcom_smd_channel *channel = qsept->qsch;  in qcom_smd_poll()  local
998 	poll_wait(filp, &channel->fblockread_event, wait);  in qcom_smd_poll()
1000 	if (qcom_smd_get_tx_avail(channel) > 20)  in qcom_smd_poll()
1010 						  const char *channel)  in qcom_smd_match_channel()  argument
1023 		if (strcmp(name, channel) == 0)  in qcom_smd_match_channel()
1033 	struct qcom_smd_channel *channel = qept->qsch;  in qcom_smd_announce_create()  local
1037 	spin_lock_irqsave(&channel->recv_lock, flags);  in qcom_smd_announce_create()
1038 	kick_state = qcom_smd_channel_intr(channel);  in qcom_smd_announce_create()
1039 	spin_unlock_irqrestore(&channel->recv_lock, flags);  in qcom_smd_announce_create()
1042 		schedule_work(&channel->edge->state_work);  in qcom_smd_announce_create()
1072 static int qcom_smd_create_device(struct qcom_smd_channel *channel)  in qcom_smd_create_device()  argument
1076 	struct qcom_smd_edge *edge = channel->edge;  in qcom_smd_create_device()
1078 	dev_dbg(&edge->dev, "registering '%s'\n", channel->name);  in qcom_smd_create_device()
1092 	strscpy_pad(rpdev->id.name, channel->name, RPMSG_NAME_SIZE);  in qcom_smd_create_device()
1096 	rpdev->dev.of_node = qcom_smd_match_channel(edge->of_node, channel->name);  in qcom_smd_create_device()
1128 	struct qcom_smd_channel *channel;  in qcom_smd_create_channel()  local
1135 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in qcom_smd_create_channel()
1136 	if (!channel)  in qcom_smd_create_channel()
1139 	channel->edge = edge;  in qcom_smd_create_channel()
1140 	channel->name = kstrdup(name, GFP_KERNEL);  in qcom_smd_create_channel()
1141 	if (!channel->name) {  in qcom_smd_create_channel()
1146 	spin_lock_init(&channel->tx_lock);  in qcom_smd_create_channel()
1147 	spin_lock_init(&channel->recv_lock);  in qcom_smd_create_channel()
1148 	init_waitqueue_head(&channel->fblockread_event);  in qcom_smd_create_channel()
1149 	init_waitqueue_head(&channel->state_change_event);  in qcom_smd_create_channel()
1162 		channel->info_word = info;  in qcom_smd_create_channel()
1164 		channel->info = info;  in qcom_smd_create_channel()
1184 	channel->tx_fifo = fifo_base;  in qcom_smd_create_channel()
1185 	channel->rx_fifo = fifo_base + fifo_size;  in qcom_smd_create_channel()
1186 	channel->fifo_size = fifo_size;  in qcom_smd_create_channel()
1188 	qcom_smd_channel_reset(channel);  in qcom_smd_create_channel()
1190 	return channel;  in qcom_smd_create_channel()
1193 	kfree(channel->name);  in qcom_smd_create_channel()
1195 	kfree(channel);  in qcom_smd_create_channel()
1210 	struct qcom_smd_channel *channel;  in qcom_channel_scan_worker()  local
1246 			channel = qcom_smd_create_channel(edge, info_id, fifo_id, entry->name);  in qcom_channel_scan_worker()
1247 			if (IS_ERR(channel))  in qcom_channel_scan_worker()
1251 			list_add(&channel->list, &edge->channels);  in qcom_channel_scan_worker()
1254 			dev_dbg(&edge->dev, "new channel found: '%s'\n", channel->name);  in qcom_channel_scan_worker()
1274 	struct qcom_smd_channel *channel;  in qcom_channel_state_worker()  local
1287 	list_for_each_entry(channel, &edge->channels, list) {  in qcom_channel_state_worker()
1288 		if (channel->state != SMD_CHANNEL_CLOSED)  in qcom_channel_state_worker()
1295 		remote_state = GET_RX_CHANNEL_INFO(channel, state);  in qcom_channel_state_worker()
1298 		    strcmp(channel->name, "rpm_requests"))  in qcom_channel_state_worker()
1301 		if (channel->registered)  in qcom_channel_state_worker()
1305 		qcom_smd_create_device(channel);  in qcom_channel_state_worker()
1307 		channel->registered = true;  in qcom_channel_state_worker()
1314 	list_for_each_entry(channel, &edge->channels, list) {  in qcom_channel_state_worker()
1315 		if (channel->state != SMD_CHANNEL_OPENING &&  in qcom_channel_state_worker()
1316 		    channel->state != SMD_CHANNEL_OPENED)  in qcom_channel_state_worker()
1319 		remote_state = GET_RX_CHANNEL_INFO(channel, state);  in qcom_channel_state_worker()
1326 		strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));  in qcom_channel_state_worker()
1330 		channel->registered = false;  in qcom_channel_state_worker()
1442 	struct qcom_smd_channel *channel, *tmp;  in qcom_smd_edge_release()  local
1445 	list_for_each_entry_safe(channel, tmp, &edge->channels, list) {  in qcom_smd_edge_release()
1446 		list_del(&channel->list);  in qcom_smd_edge_release()
1447 		kfree(channel->name);  in qcom_smd_edge_release()
1448 		kfree(channel);  in qcom_smd_edge_release()