217 	struct glink_channel *channel;  in qcom_glink_alloc_channel()  local
219 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in qcom_glink_alloc_channel()
220 	if (!channel)  in qcom_glink_alloc_channel()
224 	spin_lock_init(&channel->recv_lock);  in qcom_glink_alloc_channel()
225 	spin_lock_init(&channel->intent_lock);  in qcom_glink_alloc_channel()
226 	mutex_init(&channel->intent_req_lock);  in qcom_glink_alloc_channel()
228 	channel->glink = glink;  in qcom_glink_alloc_channel()
229 	channel->name = kstrdup(name, GFP_KERNEL);  in qcom_glink_alloc_channel()
230 	if (!channel->name) {  in qcom_glink_alloc_channel()
231 		kfree(channel);  in qcom_glink_alloc_channel()
235 	init_completion(&channel->open_req);  in qcom_glink_alloc_channel()
236 	init_completion(&channel->open_ack);  in qcom_glink_alloc_channel()
237 	init_waitqueue_head(&channel->intent_req_wq);  in qcom_glink_alloc_channel()
239 	INIT_LIST_HEAD(&channel->done_intents);  in qcom_glink_alloc_channel()
240 	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);  in qcom_glink_alloc_channel()
242 	idr_init(&channel->liids);  in qcom_glink_alloc_channel()
243 	idr_init(&channel->riids);  in qcom_glink_alloc_channel()
244 	kref_init(&channel->refcount);  in qcom_glink_alloc_channel()
246 	return channel;  in qcom_glink_alloc_channel()
251 	struct glink_channel *channel = container_of(ref, struct glink_channel,  in qcom_glink_channel_release()  local
259 	cancel_work_sync(&channel->intent_work);  in qcom_glink_channel_release()
261 	spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_channel_release()
263 	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {  in qcom_glink_channel_release()
270 	idr_for_each_entry(&channel->liids, tmp, iid) {  in qcom_glink_channel_release()
274 	idr_destroy(&channel->liids);  in qcom_glink_channel_release()
276 	idr_for_each_entry(&channel->riids, tmp, iid)  in qcom_glink_channel_release()
278 	idr_destroy(&channel->riids);  in qcom_glink_channel_release()
279 	spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_channel_release()
281 	kfree(channel->name);  in qcom_glink_channel_release()
282 	kfree(channel);  in qcom_glink_channel_release()
410 				     struct glink_channel *channel)  in qcom_glink_send_open_ack()  argument
415 	msg.param1 = cpu_to_le16(channel->rcid);  in qcom_glink_send_open_ack()
424 	struct glink_channel *channel;  in qcom_glink_handle_intent_req_ack()  local
428 	channel = idr_find(&glink->rcids, cid);  in qcom_glink_handle_intent_req_ack()
430 	if (!channel) {  in qcom_glink_handle_intent_req_ack()
435 	WRITE_ONCE(channel->intent_req_result, granted);  in qcom_glink_handle_intent_req_ack()
436 	wake_up_all(&channel->intent_req_wq);  in qcom_glink_handle_intent_req_ack()
439 static void qcom_glink_intent_req_abort(struct glink_channel *channel)  in qcom_glink_intent_req_abort()  argument
441 	WRITE_ONCE(channel->intent_req_result, 0);  in qcom_glink_intent_req_abort()
442 	wake_up_all(&channel->intent_req_wq);  in qcom_glink_intent_req_abort()
456 				    struct glink_channel *channel)  in qcom_glink_send_open_req()  argument
462 	int name_len = strlen(channel->name) + 1;  in qcom_glink_send_open_req()
467 	kref_get(&channel->refcount);  in qcom_glink_send_open_req()
470 	ret = idr_alloc_cyclic(&glink->lcids, channel,  in qcom_glink_send_open_req()
477 	channel->lcid = ret;  in qcom_glink_send_open_req()
480 	req.msg.param1 = cpu_to_le16(channel->lcid);  in qcom_glink_send_open_req()
482 	strcpy(req.name, channel->name);  in qcom_glink_send_open_req()
492 	idr_remove(&glink->lcids, channel->lcid);  in qcom_glink_send_open_req()
493 	channel->lcid = 0;  in qcom_glink_send_open_req()
500 				      struct glink_channel *channel)  in qcom_glink_send_close_req()  argument
505 	req.param1 = cpu_to_le16(channel->lcid);  in qcom_glink_send_close_req()
525 	struct glink_channel *channel = container_of(work, struct glink_channel,  in qcom_glink_rx_done_work()  local
527 	struct qcom_glink *glink = channel->glink;  in qcom_glink_rx_done_work()
535 	unsigned int cid = channel->lcid;  in qcom_glink_rx_done_work()
540 	spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_rx_done_work()
541 	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {  in qcom_glink_rx_done_work()
543 		spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_rx_done_work()
556 		spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_rx_done_work()
558 	spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_rx_done_work()
562 			       struct glink_channel *channel,  in qcom_glink_rx_done()  argument
574 		spin_lock(&channel->intent_lock);  in qcom_glink_rx_done()
575 		idr_remove(&channel->liids, intent->id);  in qcom_glink_rx_done()
576 		spin_unlock(&channel->intent_lock);  in qcom_glink_rx_done()
580 	spin_lock(&channel->intent_lock);  in qcom_glink_rx_done()
581 	list_add_tail(&intent->node, &channel->done_intents);  in qcom_glink_rx_done()
582 	spin_unlock(&channel->intent_lock);  in qcom_glink_rx_done()
584 	schedule_work(&channel->intent_work);  in qcom_glink_rx_done()
654 					  struct glink_channel *channel,  in qcom_glink_send_intent_req_ack()  argument
660 	msg.param1 = cpu_to_le16(channel->lcid);  in qcom_glink_send_intent_req_ack()
678 				       struct glink_channel *channel,  in qcom_glink_advertise_intent()  argument
691 	cmd.lcid = cpu_to_le16(channel->lcid);  in qcom_glink_advertise_intent()
703 			struct glink_channel *channel,  in qcom_glink_alloc_intent()  argument
719 	spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_alloc_intent()
720 	ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);  in qcom_glink_alloc_intent()
722 		spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_alloc_intent()
725 	spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_alloc_intent()
745 	struct glink_channel *channel;  in qcom_glink_handle_rx_done()  local
749 	channel = idr_find(&glink->rcids, cid);  in qcom_glink_handle_rx_done()
751 	if (!channel) {  in qcom_glink_handle_rx_done()
756 	spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_handle_rx_done()
757 	intent = idr_find(&channel->riids, iid);  in qcom_glink_handle_rx_done()
760 		spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_handle_rx_done()
768 		idr_remove(&channel->riids, intent->id);  in qcom_glink_handle_rx_done()
771 	spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_handle_rx_done()
774 		WRITE_ONCE(channel->intent_received, true);  in qcom_glink_handle_rx_done()
775 		wake_up_all(&channel->intent_req_wq);  in qcom_glink_handle_rx_done()
793 	struct glink_channel *channel;  in qcom_glink_handle_intent_req()  local
797 	channel = idr_find(&glink->rcids, cid);  in qcom_glink_handle_intent_req()
800 	if (!channel) {  in qcom_glink_handle_intent_req()
805 	intent = qcom_glink_alloc_intent(glink, channel, size, false);  in qcom_glink_handle_intent_req()
807 		qcom_glink_advertise_intent(glink, channel, intent);  in qcom_glink_handle_intent_req()
809 	qcom_glink_send_intent_req_ack(glink, channel, !!intent);  in qcom_glink_handle_intent_req()
844 	struct glink_channel *channel;  in qcom_glink_rx_data()  local
873 	channel = idr_find(&glink->rcids, rcid);  in qcom_glink_rx_data()
875 	if (!channel) {  in qcom_glink_rx_data()
884 		if (!channel->buf) {  in qcom_glink_rx_data()
900 			channel->buf = intent;  in qcom_glink_rx_data()
902 			intent = channel->buf;  in qcom_glink_rx_data()
907 		spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_rx_data()
908 		intent = idr_find(&channel->liids, liid);  in qcom_glink_rx_data()
909 		spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_rx_data()
914 				channel->name, liid);  in qcom_glink_rx_data()
933 		spin_lock(&channel->recv_lock);  in qcom_glink_rx_data()
934 		if (channel->ept.cb) {  in qcom_glink_rx_data()
935 			channel->ept.cb(channel->ept.rpdev,  in qcom_glink_rx_data()
938 					channel->ept.priv,  in qcom_glink_rx_data()
941 		spin_unlock(&channel->recv_lock);  in qcom_glink_rx_data()
944 		channel->buf = NULL;  in qcom_glink_rx_data()
946 		qcom_glink_rx_done(glink, channel, intent);  in qcom_glink_rx_data()
961 	struct glink_channel *channel;  in qcom_glink_handle_intent()  local
983 	channel = idr_find(&glink->rcids, cid);  in qcom_glink_handle_intent()
985 	if (!channel) {  in qcom_glink_handle_intent()
1005 		spin_lock_irqsave(&channel->intent_lock, flags);  in qcom_glink_handle_intent()
1006 		ret = idr_alloc(&channel->riids, intent,  in qcom_glink_handle_intent()
1008 		spin_unlock_irqrestore(&channel->intent_lock, flags);  in qcom_glink_handle_intent()
1014 	WRITE_ONCE(channel->intent_received, true);  in qcom_glink_handle_intent()
1015 	wake_up_all(&channel->intent_req_wq);  in qcom_glink_handle_intent()
1023 	struct glink_channel *channel;  in qcom_glink_rx_open_ack()  local
1026 	channel = idr_find(&glink->lcids, lcid);  in qcom_glink_rx_open_ack()
1028 	if (!channel) {  in qcom_glink_rx_open_ack()
1033 	complete_all(&channel->open_ack);  in qcom_glink_rx_open_ack()
1048 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_set_flow_control()  local
1049 	struct qcom_glink *glink = channel->glink;  in qcom_glink_set_flow_control()
1057 	msg.param1 = cpu_to_le16(channel->lcid);  in qcom_glink_set_flow_control()
1066 	struct glink_channel *channel;  in qcom_glink_handle_signals()  local
1071 	channel = idr_find(&glink->rcids, rcid);  in qcom_glink_handle_signals()
1073 	if (!channel) {  in qcom_glink_handle_signals()
1080 	if (channel->ept.flow_cb)  in qcom_glink_handle_signals()
1081 		channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);  in qcom_glink_handle_signals()
1165 	struct glink_channel *channel;  in qcom_glink_create_local()  local
1169 	channel = qcom_glink_alloc_channel(glink, name);  in qcom_glink_create_local()
1170 	if (IS_ERR(channel))  in qcom_glink_create_local()
1171 		return ERR_CAST(channel);  in qcom_glink_create_local()
1173 	ret = qcom_glink_send_open_req(glink, channel);  in qcom_glink_create_local()
1177 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);  in qcom_glink_create_local()
1181 	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);  in qcom_glink_create_local()
1185 	qcom_glink_send_open_ack(glink, channel);  in qcom_glink_create_local()
1187 	return channel;  in qcom_glink_create_local()
1192 	idr_remove(&glink->lcids, channel->lcid);  in qcom_glink_create_local()
1197 	kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_create_local()
1199 	kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_create_local()
1206 				    struct glink_channel *channel)  in qcom_glink_create_remote()  argument
1210 	qcom_glink_send_open_ack(glink, channel);  in qcom_glink_create_remote()
1212 	ret = qcom_glink_send_open_req(glink, channel);  in qcom_glink_create_remote()
1216 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);  in qcom_glink_create_remote()
1231 	qcom_glink_send_close_req(glink, channel);  in qcom_glink_create_remote()
1243 	struct glink_channel *channel;  in qcom_glink_create_ept()  local
1252 	idr_for_each_entry(&glink->rcids, channel, cid) {  in qcom_glink_create_ept()
1253 		if (!strcmp(channel->name, name))  in qcom_glink_create_ept()
1258 	if (!channel) {  in qcom_glink_create_ept()
1259 		channel = qcom_glink_create_local(glink, name);  in qcom_glink_create_ept()
1260 		if (IS_ERR(channel))  in qcom_glink_create_ept()
1263 		ret = qcom_glink_create_remote(glink, channel);  in qcom_glink_create_ept()
1268 	ept = &channel->ept;  in qcom_glink_create_ept()
1279 	struct glink_channel *channel = to_glink_channel(rpdev->ept);  in qcom_glink_announce_create()  local
1281 	struct qcom_glink *glink = channel->glink;  in qcom_glink_announce_create()
1290 	if (glink->intentless || !completion_done(&channel->open_ack))  in qcom_glink_announce_create()
1304 			intent = qcom_glink_alloc_intent(glink, channel, size,  in qcom_glink_announce_create()
1309 			qcom_glink_advertise_intent(glink, channel, intent);  in qcom_glink_announce_create()
1317 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_destroy_ept()  local
1318 	struct qcom_glink *glink = channel->glink;  in qcom_glink_destroy_ept()
1321 	spin_lock_irqsave(&channel->recv_lock, flags);  in qcom_glink_destroy_ept()
1322 	channel->ept.cb = NULL;  in qcom_glink_destroy_ept()
1323 	spin_unlock_irqrestore(&channel->recv_lock, flags);  in qcom_glink_destroy_ept()
1326 	channel->rpdev = NULL;  in qcom_glink_destroy_ept()
1328 	qcom_glink_send_close_req(glink, channel);  in qcom_glink_destroy_ept()
1332 				     struct glink_channel *channel,  in qcom_glink_request_intent()  argument
1343 	mutex_lock(&channel->intent_req_lock);  in qcom_glink_request_intent()
1345 	WRITE_ONCE(channel->intent_req_result, -1);  in qcom_glink_request_intent()
1346 	WRITE_ONCE(channel->intent_received, false);  in qcom_glink_request_intent()
1349 	cmd.cid = channel->lcid;  in qcom_glink_request_intent()
1356 	ret = wait_event_timeout(channel->intent_req_wq,  in qcom_glink_request_intent()
1357 				 READ_ONCE(channel->intent_req_result) >= 0 &&  in qcom_glink_request_intent()
1358 				 READ_ONCE(channel->intent_received),  in qcom_glink_request_intent()
1364 		ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED;  in qcom_glink_request_intent()
1368 	mutex_unlock(&channel->intent_req_lock);  in qcom_glink_request_intent()
1372 static int __qcom_glink_send(struct glink_channel *channel,  in __qcom_glink_send()  argument
1375 	struct qcom_glink *glink = channel->glink;  in __qcom_glink_send()
1391 			spin_lock_irqsave(&channel->intent_lock, flags);  in __qcom_glink_send()
1392 			idr_for_each_entry(&channel->riids, tmp, iid) {  in __qcom_glink_send()
1404 			spin_unlock_irqrestore(&channel->intent_lock, flags);  in __qcom_glink_send()
1413 			ret = qcom_glink_request_intent(glink, channel, len);  in __qcom_glink_send()
1427 		req.msg.param1 = cpu_to_le16(channel->lcid);  in __qcom_glink_send()
1448 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_send()  local
1450 	return __qcom_glink_send(channel, data, len, true);  in qcom_glink_send()
1455 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_trysend()  local
1457 	return __qcom_glink_send(channel, data, len, false);  in qcom_glink_trysend()
1462 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_sendto()  local
1464 	return __qcom_glink_send(channel, data, len, true);  in qcom_glink_sendto()
1469 	struct glink_channel *channel = to_glink_channel(ept);  in qcom_glink_trysendto()  local
1471 	return __qcom_glink_send(channel, data, len, false);  in qcom_glink_trysendto()
1478 						    const char *channel)  in qcom_glink_match_channel()  argument
1491 		if (strcmp(name, channel) == 0)  in qcom_glink_match_channel()
1523 	struct glink_channel *channel;  in qcom_glink_rx_open()  local
1532 	idr_for_each_entry(&glink->lcids, channel, lcid) {  in qcom_glink_rx_open()
1533 		if (!strcmp(channel->name, name))  in qcom_glink_rx_open()
1538 	if (!channel) {  in qcom_glink_rx_open()
1539 		channel = qcom_glink_alloc_channel(glink, name);  in qcom_glink_rx_open()
1540 		if (IS_ERR(channel))  in qcom_glink_rx_open()
1541 			return PTR_ERR(channel);  in qcom_glink_rx_open()
1548 	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);  in qcom_glink_rx_open()
1554 	channel->rcid = ret;  in qcom_glink_rx_open()
1557 	complete_all(&channel->open_req);  in qcom_glink_rx_open()
1566 		rpdev->ept = &channel->ept;  in qcom_glink_rx_open()
1581 		channel->rpdev = rpdev;  in qcom_glink_rx_open()
1588 	idr_remove(&glink->rcids, channel->rcid);  in qcom_glink_rx_open()
1589 	channel->rcid = 0;  in qcom_glink_rx_open()
1594 		kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_rx_open()
1602 	struct glink_channel *channel;  in qcom_glink_rx_close()  local
1606 	channel = idr_find(&glink->rcids, rcid);  in qcom_glink_rx_close()
1608 	if (WARN(!channel, "close request on unknown channel\n"))  in qcom_glink_rx_close()
1612 	cancel_work_sync(&channel->intent_work);  in qcom_glink_rx_close()
1614 	if (channel->rpdev) {  in qcom_glink_rx_close()
1615 		strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));  in qcom_glink_rx_close()
1621 	channel->rpdev = NULL;  in qcom_glink_rx_close()
1623 	qcom_glink_send_close_ack(glink, channel->rcid);  in qcom_glink_rx_close()
1626 	idr_remove(&glink->rcids, channel->rcid);  in qcom_glink_rx_close()
1627 	channel->rcid = 0;  in qcom_glink_rx_close()
1630 	kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_rx_close()
1636 	struct glink_channel *channel;  in qcom_glink_rx_close_ack()  local
1643 	channel = idr_find(&glink->lcids, lcid);  in qcom_glink_rx_close_ack()
1644 	if (WARN(!channel, "close ack on unknown channel\n")) {  in qcom_glink_rx_close_ack()
1649 	idr_remove(&glink->lcids, channel->lcid);  in qcom_glink_rx_close_ack()
1650 	channel->lcid = 0;  in qcom_glink_rx_close_ack()
1654 	if (channel->rpdev) {  in qcom_glink_rx_close_ack()
1655 		strscpy(chinfo.name, channel->name, sizeof(chinfo.name));  in qcom_glink_rx_close_ack()
1661 	channel->rpdev = NULL;  in qcom_glink_rx_close_ack()
1663 	kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_rx_close_ack()
1756 	struct glink_channel *channel = to_glink_channel(rpdev->ept);  in qcom_glink_device_release()  local
1759 	kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_device_release()
1767 	struct glink_channel *channel;  in qcom_glink_create_chrdev()  local
1773 	channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");  in qcom_glink_create_chrdev()
1774 	if (IS_ERR(channel)) {  in qcom_glink_create_chrdev()
1776 		return PTR_ERR(channel);  in qcom_glink_create_chrdev()
1778 	channel->rpdev = rpdev;  in qcom_glink_create_chrdev()
1780 	rpdev->ept = &channel->ept;  in qcom_glink_create_chrdev()
1845 	struct glink_channel *channel;  in qcom_glink_native_remove()  local
1860 	idr_for_each_entry(&glink->lcids, channel, cid)  in qcom_glink_native_remove()
1861 		qcom_glink_intent_req_abort(channel);  in qcom_glink_native_remove()
1869 	idr_for_each_entry(&glink->lcids, channel, cid)  in qcom_glink_native_remove()
1870 		kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_native_remove()
1873 	idr_for_each_entry(&glink->rcids, channel, cid)  in qcom_glink_native_remove()
1874 		kref_put(&channel->refcount, qcom_glink_channel_release);  in qcom_glink_native_remove()