1 /*
2  * Copyright (c) 2016-2017, Linaro Ltd
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 and
6  * only version 2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  */
13 
14 #include <linux/idr.h>
15 #include <linux/interrupt.h>
16 #include <linux/io.h>
17 #include <linux/list.h>
18 #include <linux/mfd/syscon.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/platform_device.h>
24 #include <linux/regmap.h>
25 #include <linux/rpmsg.h>
26 #include <linux/sizes.h>
27 #include <linux/slab.h>
28 #include <linux/workqueue.h>
29 #include <linux/mailbox_client.h>
30 
31 #include "rpmsg_internal.h"
32 #include "qcom_glink_native.h"
33 
34 #define GLINK_NAME_SIZE		32
35 #define GLINK_VERSION_1		1
36 
37 #define RPM_GLINK_CID_MIN	1
38 #define RPM_GLINK_CID_MAX	65536
39 
40 struct glink_msg {
41 	__le16 cmd;
42 	__le16 param1;
43 	__le32 param2;
44 	u8 data[];
45 } __packed;
46 
47 /**
48  * struct glink_defer_cmd - deferred incoming control message
49  * @node:	list node
50  * @msg:	message header
51  * data:	payload of the message
52  *
53  * Copy of a received control message, to be added to @rx_queue and processed
54  * by @rx_work of @qcom_glink.
55  */
56 struct glink_defer_cmd {
57 	struct list_head node;
58 
59 	struct glink_msg msg;
60 	u8 data[];
61 };
62 
63 /**
64  * struct glink_core_rx_intent - RX intent
65  * RX intent
66  *
67  * data: pointer to the data (may be NULL for zero-copy)
68  * id: remote or local intent ID
69  * size: size of the original intent (do not modify)
70  * reuse: To mark if the intent can be reused after first use
71  * in_use: To mark if intent is already in use for the channel
72  * offset: next write offset (initially 0)
73  */
74 struct glink_core_rx_intent {
75 	void *data;
76 	u32 id;
77 	size_t size;
78 	bool reuse;
79 	bool in_use;
80 	u32 offset;
81 
82 	struct list_head node;
83 };
84 
85 /**
86  * struct qcom_glink - driver context, relates to one remote subsystem
87  * @dev:	reference to the associated struct device
88  * @mbox_client: mailbox client
89  * @mbox_chan:  mailbox channel
90  * @rx_pipe:	pipe object for receive FIFO
91  * @tx_pipe:	pipe object for transmit FIFO
92  * @irq:	IRQ for signaling incoming events
93  * @rx_work:	worker for handling received control messages
94  * @rx_lock:	protects the @rx_queue
95  * @rx_queue:	queue of received control messages to be processed in @rx_work
96  * @tx_lock:	synchronizes operations on the tx fifo
97  * @idr_lock:	synchronizes @lcids and @rcids modifications
98  * @lcids:	idr of all channels with a known local channel id
99  * @rcids:	idr of all channels with a known remote channel id
100  */
101 struct qcom_glink {
102 	struct device *dev;
103 
104 	struct mbox_client mbox_client;
105 	struct mbox_chan *mbox_chan;
106 
107 	struct qcom_glink_pipe *rx_pipe;
108 	struct qcom_glink_pipe *tx_pipe;
109 
110 	int irq;
111 
112 	struct work_struct rx_work;
113 	spinlock_t rx_lock;
114 	struct list_head rx_queue;
115 
116 	struct mutex tx_lock;
117 
118 	spinlock_t idr_lock;
119 	struct idr lcids;
120 	struct idr rcids;
121 	unsigned long features;
122 
123 	bool intentless;
124 };
125 
126 enum {
127 	GLINK_STATE_CLOSED,
128 	GLINK_STATE_OPENING,
129 	GLINK_STATE_OPEN,
130 	GLINK_STATE_CLOSING,
131 };
132 
133 /**
134  * struct glink_channel - internal representation of a channel
135  * @rpdev:	rpdev reference, only used for primary endpoints
136  * @ept:	rpmsg endpoint this channel is associated with
137  * @glink:	qcom_glink context handle
138  * @refcount:	refcount for the channel object
139  * @recv_lock:	guard for @ept.cb
140  * @name:	unique channel name/identifier
141  * @lcid:	channel id, in local space
142  * @rcid:	channel id, in remote space
143  * @intent_lock: lock for protection of @liids, @riids
144  * @liids:	idr of all local intents
145  * @riids:	idr of all remote intents
146  * @intent_work: worker responsible for transmitting rx_done packets
147  * @done_intents: list of intents that needs to be announced rx_done
148  * @buf:	receive buffer, for gathering fragments
149  * @buf_offset:	write offset in @buf
150  * @buf_size:	size of current @buf
151  * @open_ack:	completed once remote has acked the open-request
152  * @open_req:	completed once open-request has been received
153  * @intent_req_lock: Synchronises multiple intent requests
154  * @intent_req_result: Result of intent request
155  * @intent_req_comp: Completion for intent_req signalling
156  */
157 struct glink_channel {
158 	struct rpmsg_endpoint ept;
159 
160 	struct rpmsg_device *rpdev;
161 	struct qcom_glink *glink;
162 
163 	struct kref refcount;
164 
165 	spinlock_t recv_lock;
166 
167 	char *name;
168 	unsigned int lcid;
169 	unsigned int rcid;
170 
171 	spinlock_t intent_lock;
172 	struct idr liids;
173 	struct idr riids;
174 	struct work_struct intent_work;
175 	struct list_head done_intents;
176 
177 	struct glink_core_rx_intent *buf;
178 	int buf_offset;
179 	int buf_size;
180 
181 	struct completion open_ack;
182 	struct completion open_req;
183 
184 	struct mutex intent_req_lock;
185 	bool intent_req_result;
186 	struct completion intent_req_comp;
187 };
188 
189 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
190 
191 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
192 
193 #define RPM_CMD_VERSION			0
194 #define RPM_CMD_VERSION_ACK		1
195 #define RPM_CMD_OPEN			2
196 #define RPM_CMD_CLOSE			3
197 #define RPM_CMD_OPEN_ACK		4
198 #define RPM_CMD_INTENT			5
199 #define RPM_CMD_RX_DONE			6
200 #define RPM_CMD_RX_INTENT_REQ		7
201 #define RPM_CMD_RX_INTENT_REQ_ACK	8
202 #define RPM_CMD_TX_DATA			9
203 #define RPM_CMD_CLOSE_ACK		11
204 #define RPM_CMD_TX_DATA_CONT		12
205 #define RPM_CMD_READ_NOTIF		13
206 #define RPM_CMD_RX_DONE_W_REUSE		14
207 
208 #define GLINK_FEATURE_INTENTLESS	BIT(1)
209 
210 static void qcom_glink_rx_done_work(struct work_struct *work);
211 
212 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
213 						      const char *name)
214 {
215 	struct glink_channel *channel;
216 
217 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
218 	if (!channel)
219 		return ERR_PTR(-ENOMEM);
220 
221 	/* Setup glink internal glink_channel data */
222 	spin_lock_init(&channel->recv_lock);
223 	spin_lock_init(&channel->intent_lock);
224 	mutex_init(&channel->intent_req_lock);
225 
226 	channel->glink = glink;
227 	channel->name = kstrdup(name, GFP_KERNEL);
228 
229 	init_completion(&channel->open_req);
230 	init_completion(&channel->open_ack);
231 	init_completion(&channel->intent_req_comp);
232 
233 	INIT_LIST_HEAD(&channel->done_intents);
234 	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
235 
236 	idr_init(&channel->liids);
237 	idr_init(&channel->riids);
238 	kref_init(&channel->refcount);
239 
240 	return channel;
241 }
242 
243 static void qcom_glink_channel_release(struct kref *ref)
244 {
245 	struct glink_channel *channel = container_of(ref, struct glink_channel,
246 						     refcount);
247 	unsigned long flags;
248 
249 	spin_lock_irqsave(&channel->intent_lock, flags);
250 	idr_destroy(&channel->liids);
251 	idr_destroy(&channel->riids);
252 	spin_unlock_irqrestore(&channel->intent_lock, flags);
253 
254 	kfree(channel->name);
255 	kfree(channel);
256 }
257 
258 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
259 {
260 	return glink->rx_pipe->avail(glink->rx_pipe);
261 }
262 
263 static void qcom_glink_rx_peak(struct qcom_glink *glink,
264 			       void *data, unsigned int offset, size_t count)
265 {
266 	glink->rx_pipe->peak(glink->rx_pipe, data, offset, count);
267 }
268 
269 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
270 {
271 	glink->rx_pipe->advance(glink->rx_pipe, count);
272 }
273 
274 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
275 {
276 	return glink->tx_pipe->avail(glink->tx_pipe);
277 }
278 
279 static void qcom_glink_tx_write(struct qcom_glink *glink,
280 				const void *hdr, size_t hlen,
281 				const void *data, size_t dlen)
282 {
283 	glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
284 }
285 
286 static int qcom_glink_tx(struct qcom_glink *glink,
287 			 const void *hdr, size_t hlen,
288 			 const void *data, size_t dlen, bool wait)
289 {
290 	unsigned int tlen = hlen + dlen;
291 	int ret;
292 
293 	/* Reject packets that are too big */
294 	if (tlen >= glink->tx_pipe->length)
295 		return -EINVAL;
296 
297 	ret = mutex_lock_interruptible(&glink->tx_lock);
298 	if (ret)
299 		return ret;
300 
301 	while (qcom_glink_tx_avail(glink) < tlen) {
302 		if (!wait) {
303 			ret = -EAGAIN;
304 			goto out;
305 		}
306 
307 		usleep_range(10000, 15000);
308 	}
309 
310 	qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
311 
312 	mbox_send_message(glink->mbox_chan, NULL);
313 	mbox_client_txdone(glink->mbox_chan, 0);
314 
315 out:
316 	mutex_unlock(&glink->tx_lock);
317 
318 	return ret;
319 }
320 
321 static int qcom_glink_send_version(struct qcom_glink *glink)
322 {
323 	struct glink_msg msg;
324 
325 	msg.cmd = cpu_to_le16(RPM_CMD_VERSION);
326 	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
327 	msg.param2 = cpu_to_le32(glink->features);
328 
329 	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
330 }
331 
332 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
333 {
334 	struct glink_msg msg;
335 
336 	msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK);
337 	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
338 	msg.param2 = cpu_to_le32(glink->features);
339 
340 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
341 }
342 
343 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
344 				     struct glink_channel *channel)
345 {
346 	struct glink_msg msg;
347 
348 	msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK);
349 	msg.param1 = cpu_to_le16(channel->rcid);
350 	msg.param2 = cpu_to_le32(0);
351 
352 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
353 }
354 
355 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
356 					     unsigned int cid, bool granted)
357 {
358 	struct glink_channel *channel;
359 	unsigned long flags;
360 
361 	spin_lock_irqsave(&glink->idr_lock, flags);
362 	channel = idr_find(&glink->rcids, cid);
363 	spin_unlock_irqrestore(&glink->idr_lock, flags);
364 	if (!channel) {
365 		dev_err(glink->dev, "unable to find channel\n");
366 		return;
367 	}
368 
369 	channel->intent_req_result = granted;
370 	complete(&channel->intent_req_comp);
371 }
372 
373 /**
374  * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote
375  * @glink: Ptr to the glink edge
376  * @channel: Ptr to the channel that the open req is sent
377  *
378  * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote.
379  * Will return with refcount held, regardless of outcome.
380  *
381  * Returns 0 on success, negative errno otherwise.
382  */
383 static int qcom_glink_send_open_req(struct qcom_glink *glink,
384 				    struct glink_channel *channel)
385 {
386 	struct {
387 		struct glink_msg msg;
388 		u8 name[GLINK_NAME_SIZE];
389 	} __packed req;
390 	int name_len = strlen(channel->name) + 1;
391 	int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
392 	int ret;
393 	unsigned long flags;
394 
395 	kref_get(&channel->refcount);
396 
397 	spin_lock_irqsave(&glink->idr_lock, flags);
398 	ret = idr_alloc_cyclic(&glink->lcids, channel,
399 			       RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
400 			       GFP_ATOMIC);
401 	spin_unlock_irqrestore(&glink->idr_lock, flags);
402 	if (ret < 0)
403 		return ret;
404 
405 	channel->lcid = ret;
406 
407 	req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN);
408 	req.msg.param1 = cpu_to_le16(channel->lcid);
409 	req.msg.param2 = cpu_to_le32(name_len);
410 	strcpy(req.name, channel->name);
411 
412 	ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
413 	if (ret)
414 		goto remove_idr;
415 
416 	return 0;
417 
418 remove_idr:
419 	spin_lock_irqsave(&glink->idr_lock, flags);
420 	idr_remove(&glink->lcids, channel->lcid);
421 	channel->lcid = 0;
422 	spin_unlock_irqrestore(&glink->idr_lock, flags);
423 
424 	return ret;
425 }
426 
427 static void qcom_glink_send_close_req(struct qcom_glink *glink,
428 				      struct glink_channel *channel)
429 {
430 	struct glink_msg req;
431 
432 	req.cmd = cpu_to_le16(RPM_CMD_CLOSE);
433 	req.param1 = cpu_to_le16(channel->lcid);
434 	req.param2 = 0;
435 
436 	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
437 }
438 
439 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
440 				      unsigned int rcid)
441 {
442 	struct glink_msg req;
443 
444 	req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK);
445 	req.param1 = cpu_to_le16(rcid);
446 	req.param2 = 0;
447 
448 	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
449 }
450 
451 static void qcom_glink_rx_done_work(struct work_struct *work)
452 {
453 	struct glink_channel *channel = container_of(work, struct glink_channel,
454 						     intent_work);
455 	struct qcom_glink *glink = channel->glink;
456 	struct glink_core_rx_intent *intent, *tmp;
457 	struct {
458 		u16 id;
459 		u16 lcid;
460 		u32 liid;
461 	} __packed cmd;
462 
463 	unsigned int cid = channel->lcid;
464 	unsigned int iid;
465 	bool reuse;
466 	unsigned long flags;
467 
468 	spin_lock_irqsave(&channel->intent_lock, flags);
469 	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
470 		list_del(&intent->node);
471 		spin_unlock_irqrestore(&channel->intent_lock, flags);
472 		iid = intent->id;
473 		reuse = intent->reuse;
474 
475 		cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE;
476 		cmd.lcid = cid;
477 		cmd.liid = iid;
478 
479 		qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
480 		if (!reuse) {
481 			kfree(intent->data);
482 			kfree(intent);
483 		}
484 		spin_lock_irqsave(&channel->intent_lock, flags);
485 	}
486 	spin_unlock_irqrestore(&channel->intent_lock, flags);
487 }
488 
489 static void qcom_glink_rx_done(struct qcom_glink *glink,
490 			       struct glink_channel *channel,
491 			       struct glink_core_rx_intent *intent)
492 {
493 	/* We don't send RX_DONE to intentless systems */
494 	if (glink->intentless) {
495 		kfree(intent->data);
496 		kfree(intent);
497 		return;
498 	}
499 
500 	/* Take it off the tree of receive intents */
501 	if (!intent->reuse) {
502 		spin_lock(&channel->intent_lock);
503 		idr_remove(&channel->liids, intent->id);
504 		spin_unlock(&channel->intent_lock);
505 	}
506 
507 	/* Schedule the sending of a rx_done indication */
508 	spin_lock(&channel->intent_lock);
509 	list_add_tail(&intent->node, &channel->done_intents);
510 	spin_unlock(&channel->intent_lock);
511 
512 	schedule_work(&channel->intent_work);
513 }
514 
515 /**
516  * qcom_glink_receive_version() - receive version/features from remote system
517  *
518  * @glink:	pointer to transport interface
519  * @r_version:	remote version
520  * @r_features:	remote features
521  *
522  * This function is called in response to a remote-initiated version/feature
523  * negotiation sequence.
524  */
525 static void qcom_glink_receive_version(struct qcom_glink *glink,
526 				       u32 version,
527 				       u32 features)
528 {
529 	switch (version) {
530 	case 0:
531 		break;
532 	case GLINK_VERSION_1:
533 		glink->features &= features;
534 		/* FALLTHROUGH */
535 	default:
536 		qcom_glink_send_version_ack(glink);
537 		break;
538 	}
539 }
540 
541 /**
542  * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
543  *
544  * @glink:	pointer to transport interface
545  * @r_version:	remote version response
546  * @r_features:	remote features response
547  *
548  * This function is called in response to a local-initiated version/feature
549  * negotiation sequence and is the counter-offer from the remote side based
550  * upon the initial version and feature set requested.
551  */
552 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
553 					   u32 version,
554 					   u32 features)
555 {
556 	switch (version) {
557 	case 0:
558 		/* Version negotiation failed */
559 		break;
560 	case GLINK_VERSION_1:
561 		if (features == glink->features)
562 			break;
563 
564 		glink->features &= features;
565 		/* FALLTHROUGH */
566 	default:
567 		qcom_glink_send_version(glink);
568 		break;
569 	}
570 }
571 
572 /**
573  * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
574 				      wire format and transmit
575  * @glink:	The transport to transmit on.
576  * @channel:	The glink channel
577  * @granted:	The request response to encode.
578  *
579  * Return: 0 on success or standard Linux error code.
580  */
581 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
582 					  struct glink_channel *channel,
583 					  bool granted)
584 {
585 	struct glink_msg msg;
586 
587 	msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK);
588 	msg.param1 = cpu_to_le16(channel->lcid);
589 	msg.param2 = cpu_to_le32(granted);
590 
591 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
592 
593 	return 0;
594 }
595 
596 /**
597  * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
598  *			   transmit
599  * @glink:	The transport to transmit on.
600  * @channel:	The local channel
601  * @size:	The intent to pass on to remote.
602  *
603  * Return: 0 on success or standard Linux error code.
604  */
605 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
606 				       struct glink_channel *channel,
607 				       struct glink_core_rx_intent *intent)
608 {
609 	struct command {
610 		u16 id;
611 		u16 lcid;
612 		u32 count;
613 		u32 size;
614 		u32 liid;
615 	} __packed;
616 	struct command cmd;
617 
618 	cmd.id = cpu_to_le16(RPM_CMD_INTENT);
619 	cmd.lcid = cpu_to_le16(channel->lcid);
620 	cmd.count = cpu_to_le32(1);
621 	cmd.size = cpu_to_le32(intent->size);
622 	cmd.liid = cpu_to_le32(intent->id);
623 
624 	qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
625 
626 	return 0;
627 }
628 
629 static struct glink_core_rx_intent *
630 qcom_glink_alloc_intent(struct qcom_glink *glink,
631 			struct glink_channel *channel,
632 			size_t size,
633 			bool reuseable)
634 {
635 	struct glink_core_rx_intent *intent;
636 	int ret;
637 	unsigned long flags;
638 
639 	intent = kzalloc(sizeof(*intent), GFP_KERNEL);
640 	if (!intent)
641 		return NULL;
642 
643 	intent->data = kzalloc(size, GFP_KERNEL);
644 	if (!intent->data)
645 		goto free_intent;
646 
647 	spin_lock_irqsave(&channel->intent_lock, flags);
648 	ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
649 	if (ret < 0) {
650 		spin_unlock_irqrestore(&channel->intent_lock, flags);
651 		goto free_data;
652 	}
653 	spin_unlock_irqrestore(&channel->intent_lock, flags);
654 
655 	intent->id = ret;
656 	intent->size = size;
657 	intent->reuse = reuseable;
658 
659 	return intent;
660 
661 free_data:
662 	kfree(intent->data);
663 free_intent:
664 	kfree(intent);
665 	return NULL;
666 }
667 
668 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
669 				      u32 cid, uint32_t iid,
670 				      bool reuse)
671 {
672 	struct glink_core_rx_intent *intent;
673 	struct glink_channel *channel;
674 	unsigned long flags;
675 
676 	spin_lock_irqsave(&glink->idr_lock, flags);
677 	channel = idr_find(&glink->rcids, cid);
678 	spin_unlock_irqrestore(&glink->idr_lock, flags);
679 	if (!channel) {
680 		dev_err(glink->dev, "invalid channel id received\n");
681 		return;
682 	}
683 
684 	spin_lock_irqsave(&channel->intent_lock, flags);
685 	intent = idr_find(&channel->riids, iid);
686 
687 	if (!intent) {
688 		spin_unlock_irqrestore(&channel->intent_lock, flags);
689 		dev_err(glink->dev, "invalid intent id received\n");
690 		return;
691 	}
692 
693 	intent->in_use = false;
694 
695 	if (!reuse) {
696 		idr_remove(&channel->riids, intent->id);
697 		kfree(intent);
698 	}
699 	spin_unlock_irqrestore(&channel->intent_lock, flags);
700 }
701 
702 /**
703  * qcom_glink_handle_intent_req() - Receive a request for rx_intent
704  *					    from remote side
705  * if_ptr:      Pointer to the transport interface
706  * rcid:	Remote channel ID
707  * size:	size of the intent
708  *
709  * The function searches for the local channel to which the request for
710  * rx_intent has arrived and allocates and notifies the remote back
711  */
712 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
713 					 u32 cid, size_t size)
714 {
715 	struct glink_core_rx_intent *intent;
716 	struct glink_channel *channel;
717 	unsigned long flags;
718 
719 	spin_lock_irqsave(&glink->idr_lock, flags);
720 	channel = idr_find(&glink->rcids, cid);
721 	spin_unlock_irqrestore(&glink->idr_lock, flags);
722 
723 	if (!channel) {
724 		pr_err("%s channel not found for cid %d\n", __func__, cid);
725 		return;
726 	}
727 
728 	intent = qcom_glink_alloc_intent(glink, channel, size, false);
729 	if (intent)
730 		qcom_glink_advertise_intent(glink, channel, intent);
731 
732 	qcom_glink_send_intent_req_ack(glink, channel, !!intent);
733 }
734 
735 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
736 {
737 	struct glink_defer_cmd *dcmd;
738 
739 	extra = ALIGN(extra, 8);
740 
741 	if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
742 		dev_dbg(glink->dev, "Insufficient data in rx fifo");
743 		return -ENXIO;
744 	}
745 
746 	dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC);
747 	if (!dcmd)
748 		return -ENOMEM;
749 
750 	INIT_LIST_HEAD(&dcmd->node);
751 
752 	qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
753 
754 	spin_lock(&glink->rx_lock);
755 	list_add_tail(&dcmd->node, &glink->rx_queue);
756 	spin_unlock(&glink->rx_lock);
757 
758 	schedule_work(&glink->rx_work);
759 	qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
760 
761 	return 0;
762 }
763 
764 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
765 {
766 	struct glink_core_rx_intent *intent;
767 	struct glink_channel *channel;
768 	struct {
769 		struct glink_msg msg;
770 		__le32 chunk_size;
771 		__le32 left_size;
772 	} __packed hdr;
773 	unsigned int chunk_size;
774 	unsigned int left_size;
775 	unsigned int rcid;
776 	unsigned int liid;
777 	int ret = 0;
778 	unsigned long flags;
779 
780 	if (avail < sizeof(hdr)) {
781 		dev_dbg(glink->dev, "Not enough data in fifo\n");
782 		return -EAGAIN;
783 	}
784 
785 	qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr));
786 	chunk_size = le32_to_cpu(hdr.chunk_size);
787 	left_size = le32_to_cpu(hdr.left_size);
788 
789 	if (avail < sizeof(hdr) + chunk_size) {
790 		dev_dbg(glink->dev, "Payload not yet in fifo\n");
791 		return -EAGAIN;
792 	}
793 
794 	if (WARN(chunk_size % 4, "Incoming data must be word aligned\n"))
795 		return -EINVAL;
796 
797 	rcid = le16_to_cpu(hdr.msg.param1);
798 	spin_lock_irqsave(&glink->idr_lock, flags);
799 	channel = idr_find(&glink->rcids, rcid);
800 	spin_unlock_irqrestore(&glink->idr_lock, flags);
801 	if (!channel) {
802 		dev_dbg(glink->dev, "Data on non-existing channel\n");
803 
804 		/* Drop the message */
805 		goto advance_rx;
806 	}
807 
808 	if (glink->intentless) {
809 		/* Might have an ongoing, fragmented, message to append */
810 		if (!channel->buf) {
811 			intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
812 			if (!intent)
813 				return -ENOMEM;
814 
815 			intent->data = kmalloc(chunk_size + left_size,
816 					       GFP_ATOMIC);
817 			if (!intent->data) {
818 				kfree(intent);
819 				return -ENOMEM;
820 			}
821 
822 			intent->id = 0xbabababa;
823 			intent->size = chunk_size + left_size;
824 			intent->offset = 0;
825 
826 			channel->buf = intent;
827 		} else {
828 			intent = channel->buf;
829 		}
830 	} else {
831 		liid = le32_to_cpu(hdr.msg.param2);
832 
833 		spin_lock_irqsave(&channel->intent_lock, flags);
834 		intent = idr_find(&channel->liids, liid);
835 		spin_unlock_irqrestore(&channel->intent_lock, flags);
836 
837 		if (!intent) {
838 			dev_err(glink->dev,
839 				"no intent found for channel %s intent %d",
840 				channel->name, liid);
841 			goto advance_rx;
842 		}
843 	}
844 
845 	if (intent->size - intent->offset < chunk_size) {
846 		dev_err(glink->dev, "Insufficient space in intent\n");
847 
848 		/* The packet header lied, drop payload */
849 		goto advance_rx;
850 	}
851 
852 	qcom_glink_rx_peak(glink, intent->data + intent->offset,
853 			   sizeof(hdr), chunk_size);
854 	intent->offset += chunk_size;
855 
856 	/* Handle message when no fragments remain to be received */
857 	if (!left_size) {
858 		spin_lock(&channel->recv_lock);
859 		if (channel->ept.cb) {
860 			channel->ept.cb(channel->ept.rpdev,
861 					intent->data,
862 					intent->offset,
863 					channel->ept.priv,
864 					RPMSG_ADDR_ANY);
865 		}
866 		spin_unlock(&channel->recv_lock);
867 
868 		intent->offset = 0;
869 		channel->buf = NULL;
870 
871 		qcom_glink_rx_done(glink, channel, intent);
872 	}
873 
874 advance_rx:
875 	qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
876 
877 	return ret;
878 }
879 
880 static void qcom_glink_handle_intent(struct qcom_glink *glink,
881 				     unsigned int cid,
882 				     unsigned int count,
883 				     size_t avail)
884 {
885 	struct glink_core_rx_intent *intent;
886 	struct glink_channel *channel;
887 	struct intent_pair {
888 		__le32 size;
889 		__le32 iid;
890 	};
891 
892 	struct {
893 		struct glink_msg msg;
894 		struct intent_pair intents[];
895 	} __packed * msg;
896 
897 	const size_t msglen = sizeof(*msg) + sizeof(struct intent_pair) * count;
898 	int ret;
899 	int i;
900 	unsigned long flags;
901 
902 	if (avail < msglen) {
903 		dev_dbg(glink->dev, "Not enough data in fifo\n");
904 		return;
905 	}
906 
907 	spin_lock_irqsave(&glink->idr_lock, flags);
908 	channel = idr_find(&glink->rcids, cid);
909 	spin_unlock_irqrestore(&glink->idr_lock, flags);
910 	if (!channel) {
911 		dev_err(glink->dev, "intents for non-existing channel\n");
912 		return;
913 	}
914 
915 	msg = kmalloc(msglen, GFP_ATOMIC);
916 	if (!msg)
917 		return;
918 
919 	qcom_glink_rx_peak(glink, msg, 0, msglen);
920 
921 	for (i = 0; i < count; ++i) {
922 		intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
923 		if (!intent)
924 			break;
925 
926 		intent->id = le32_to_cpu(msg->intents[i].iid);
927 		intent->size = le32_to_cpu(msg->intents[i].size);
928 
929 		spin_lock_irqsave(&channel->intent_lock, flags);
930 		ret = idr_alloc(&channel->riids, intent,
931 				intent->id, intent->id + 1, GFP_ATOMIC);
932 		spin_unlock_irqrestore(&channel->intent_lock, flags);
933 
934 		if (ret < 0)
935 			dev_err(glink->dev, "failed to store remote intent\n");
936 	}
937 
938 	kfree(msg);
939 	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
940 }
941 
942 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
943 {
944 	struct glink_channel *channel;
945 
946 	spin_lock(&glink->idr_lock);
947 	channel = idr_find(&glink->lcids, lcid);
948 	spin_unlock(&glink->idr_lock);
949 	if (!channel) {
950 		dev_err(glink->dev, "Invalid open ack packet\n");
951 		return -EINVAL;
952 	}
953 
954 	complete(&channel->open_ack);
955 
956 	return 0;
957 }
958 
959 static irqreturn_t qcom_glink_native_intr(int irq, void *data)
960 {
961 	struct qcom_glink *glink = data;
962 	struct glink_msg msg;
963 	unsigned int param1;
964 	unsigned int param2;
965 	unsigned int avail;
966 	unsigned int cmd;
967 	int ret = 0;
968 
969 	for (;;) {
970 		avail = qcom_glink_rx_avail(glink);
971 		if (avail < sizeof(msg))
972 			break;
973 
974 		qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg));
975 
976 		cmd = le16_to_cpu(msg.cmd);
977 		param1 = le16_to_cpu(msg.param1);
978 		param2 = le32_to_cpu(msg.param2);
979 
980 		switch (cmd) {
981 		case RPM_CMD_VERSION:
982 		case RPM_CMD_VERSION_ACK:
983 		case RPM_CMD_CLOSE:
984 		case RPM_CMD_CLOSE_ACK:
985 		case RPM_CMD_RX_INTENT_REQ:
986 			ret = qcom_glink_rx_defer(glink, 0);
987 			break;
988 		case RPM_CMD_OPEN_ACK:
989 			ret = qcom_glink_rx_open_ack(glink, param1);
990 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
991 			break;
992 		case RPM_CMD_OPEN:
993 			ret = qcom_glink_rx_defer(glink, param2);
994 			break;
995 		case RPM_CMD_TX_DATA:
996 		case RPM_CMD_TX_DATA_CONT:
997 			ret = qcom_glink_rx_data(glink, avail);
998 			break;
999 		case RPM_CMD_READ_NOTIF:
1000 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1001 
1002 			mbox_send_message(glink->mbox_chan, NULL);
1003 			mbox_client_txdone(glink->mbox_chan, 0);
1004 			break;
1005 		case RPM_CMD_INTENT:
1006 			qcom_glink_handle_intent(glink, param1, param2, avail);
1007 			break;
1008 		case RPM_CMD_RX_DONE:
1009 			qcom_glink_handle_rx_done(glink, param1, param2, false);
1010 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1011 			break;
1012 		case RPM_CMD_RX_DONE_W_REUSE:
1013 			qcom_glink_handle_rx_done(glink, param1, param2, true);
1014 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1015 			break;
1016 		case RPM_CMD_RX_INTENT_REQ_ACK:
1017 			qcom_glink_handle_intent_req_ack(glink, param1, param2);
1018 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1019 			break;
1020 		default:
1021 			dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1022 			ret = -EINVAL;
1023 			break;
1024 		}
1025 
1026 		if (ret)
1027 			break;
1028 	}
1029 
1030 	return IRQ_HANDLED;
1031 }
1032 
1033 /* Locally initiated rpmsg_create_ept */
1034 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1035 						     const char *name)
1036 {
1037 	struct glink_channel *channel;
1038 	int ret;
1039 	unsigned long flags;
1040 
1041 	channel = qcom_glink_alloc_channel(glink, name);
1042 	if (IS_ERR(channel))
1043 		return ERR_CAST(channel);
1044 
1045 	ret = qcom_glink_send_open_req(glink, channel);
1046 	if (ret)
1047 		goto release_channel;
1048 
1049 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1050 	if (!ret)
1051 		goto err_timeout;
1052 
1053 	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1054 	if (!ret)
1055 		goto err_timeout;
1056 
1057 	qcom_glink_send_open_ack(glink, channel);
1058 
1059 	return channel;
1060 
1061 err_timeout:
1062 	/* qcom_glink_send_open_req() did register the channel in lcids*/
1063 	spin_lock_irqsave(&glink->idr_lock, flags);
1064 	idr_remove(&glink->lcids, channel->lcid);
1065 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1066 
1067 release_channel:
1068 	/* Release qcom_glink_send_open_req() reference */
1069 	kref_put(&channel->refcount, qcom_glink_channel_release);
1070 	/* Release qcom_glink_alloc_channel() reference */
1071 	kref_put(&channel->refcount, qcom_glink_channel_release);
1072 
1073 	return ERR_PTR(-ETIMEDOUT);
1074 }
1075 
1076 /* Remote initiated rpmsg_create_ept */
1077 static int qcom_glink_create_remote(struct qcom_glink *glink,
1078 				    struct glink_channel *channel)
1079 {
1080 	int ret;
1081 
1082 	qcom_glink_send_open_ack(glink, channel);
1083 
1084 	ret = qcom_glink_send_open_req(glink, channel);
1085 	if (ret)
1086 		goto close_link;
1087 
1088 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1089 	if (!ret) {
1090 		ret = -ETIMEDOUT;
1091 		goto close_link;
1092 	}
1093 
1094 	return 0;
1095 
1096 close_link:
1097 	/*
1098 	 * Send a close request to "undo" our open-ack. The close-ack will
1099 	 * release the last reference.
1100 	 */
1101 	qcom_glink_send_close_req(glink, channel);
1102 
1103 	/* Release qcom_glink_send_open_req() reference */
1104 	kref_put(&channel->refcount, qcom_glink_channel_release);
1105 
1106 	return ret;
1107 }
1108 
1109 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1110 						    rpmsg_rx_cb_t cb,
1111 						    void *priv,
1112 						    struct rpmsg_channel_info
1113 									chinfo)
1114 {
1115 	struct glink_channel *parent = to_glink_channel(rpdev->ept);
1116 	struct glink_channel *channel;
1117 	struct qcom_glink *glink = parent->glink;
1118 	struct rpmsg_endpoint *ept;
1119 	const char *name = chinfo.name;
1120 	int cid;
1121 	int ret;
1122 	unsigned long flags;
1123 
1124 	spin_lock_irqsave(&glink->idr_lock, flags);
1125 	idr_for_each_entry(&glink->rcids, channel, cid) {
1126 		if (!strcmp(channel->name, name))
1127 			break;
1128 	}
1129 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1130 
1131 	if (!channel) {
1132 		channel = qcom_glink_create_local(glink, name);
1133 		if (IS_ERR(channel))
1134 			return NULL;
1135 	} else {
1136 		ret = qcom_glink_create_remote(glink, channel);
1137 		if (ret)
1138 			return NULL;
1139 	}
1140 
1141 	ept = &channel->ept;
1142 	ept->rpdev = rpdev;
1143 	ept->cb = cb;
1144 	ept->priv = priv;
1145 	ept->ops = &glink_endpoint_ops;
1146 
1147 	return ept;
1148 }
1149 
1150 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1151 {
1152 	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1153 	struct device_node *np = rpdev->dev.of_node;
1154 	struct qcom_glink *glink = channel->glink;
1155 	struct glink_core_rx_intent *intent;
1156 	const struct property *prop = NULL;
1157 	__be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1158 	int num_intents;
1159 	int num_groups = 1;
1160 	__be32 *val = defaults;
1161 	int size;
1162 
1163 	if (glink->intentless)
1164 		return 0;
1165 
1166 	prop = of_find_property(np, "qcom,intents", NULL);
1167 	if (prop) {
1168 		val = prop->value;
1169 		num_groups = prop->length / sizeof(u32) / 2;
1170 	}
1171 
1172 	/* Channel is now open, advertise base set of intents */
1173 	while (num_groups--) {
1174 		size = be32_to_cpup(val++);
1175 		num_intents = be32_to_cpup(val++);
1176 		while (num_intents--) {
1177 			intent = qcom_glink_alloc_intent(glink, channel, size,
1178 							 true);
1179 			if (!intent)
1180 				break;
1181 
1182 			qcom_glink_advertise_intent(glink, channel, intent);
1183 		}
1184 	}
1185 	return 0;
1186 }
1187 
1188 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1189 {
1190 	struct glink_channel *channel = to_glink_channel(ept);
1191 	struct qcom_glink *glink = channel->glink;
1192 	unsigned long flags;
1193 
1194 	spin_lock_irqsave(&channel->recv_lock, flags);
1195 	channel->ept.cb = NULL;
1196 	spin_unlock_irqrestore(&channel->recv_lock, flags);
1197 
1198 	/* Decouple the potential rpdev from the channel */
1199 	channel->rpdev = NULL;
1200 
1201 	qcom_glink_send_close_req(glink, channel);
1202 }
1203 
1204 static int qcom_glink_request_intent(struct qcom_glink *glink,
1205 				     struct glink_channel *channel,
1206 				     size_t size)
1207 {
1208 	struct {
1209 		u16 id;
1210 		u16 cid;
1211 		u32 size;
1212 	} __packed cmd;
1213 
1214 	int ret;
1215 
1216 	mutex_lock(&channel->intent_req_lock);
1217 
1218 	reinit_completion(&channel->intent_req_comp);
1219 
1220 	cmd.id = RPM_CMD_RX_INTENT_REQ;
1221 	cmd.cid = channel->lcid;
1222 	cmd.size = size;
1223 
1224 	ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1225 	if (ret)
1226 		goto unlock;
1227 
1228 	ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ);
1229 	if (!ret) {
1230 		dev_err(glink->dev, "intent request timed out\n");
1231 		ret = -ETIMEDOUT;
1232 	} else {
1233 		ret = channel->intent_req_result ? 0 : -ECANCELED;
1234 	}
1235 
1236 unlock:
1237 	mutex_unlock(&channel->intent_req_lock);
1238 	return ret;
1239 }
1240 
1241 static int __qcom_glink_send(struct glink_channel *channel,
1242 			     void *data, int len, bool wait)
1243 {
1244 	struct qcom_glink *glink = channel->glink;
1245 	struct glink_core_rx_intent *intent = NULL;
1246 	struct glink_core_rx_intent *tmp;
1247 	int iid = 0;
1248 	struct {
1249 		struct glink_msg msg;
1250 		__le32 chunk_size;
1251 		__le32 left_size;
1252 	} __packed req;
1253 	int ret;
1254 	unsigned long flags;
1255 
1256 	if (!glink->intentless) {
1257 		while (!intent) {
1258 			spin_lock_irqsave(&channel->intent_lock, flags);
1259 			idr_for_each_entry(&channel->riids, tmp, iid) {
1260 				if (tmp->size >= len && !tmp->in_use) {
1261 					if (!intent)
1262 						intent = tmp;
1263 					else if (intent->size > tmp->size)
1264 						intent = tmp;
1265 					if (intent->size == len)
1266 						break;
1267 				}
1268 			}
1269 			if (intent)
1270 				intent->in_use = true;
1271 			spin_unlock_irqrestore(&channel->intent_lock, flags);
1272 
1273 			/* We found an available intent */
1274 			if (intent)
1275 				break;
1276 
1277 			if (!wait)
1278 				return -EBUSY;
1279 
1280 			ret = qcom_glink_request_intent(glink, channel, len);
1281 			if (ret < 0)
1282 				return ret;
1283 		}
1284 
1285 		iid = intent->id;
1286 	}
1287 
1288 	req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA);
1289 	req.msg.param1 = cpu_to_le16(channel->lcid);
1290 	req.msg.param2 = cpu_to_le32(iid);
1291 	req.chunk_size = cpu_to_le32(len);
1292 	req.left_size = cpu_to_le32(0);
1293 
1294 	ret = qcom_glink_tx(glink, &req, sizeof(req), data, len, wait);
1295 
1296 	/* Mark intent available if we failed */
1297 	if (ret && intent)
1298 		intent->in_use = false;
1299 
1300 	return ret;
1301 }
1302 
1303 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1304 {
1305 	struct glink_channel *channel = to_glink_channel(ept);
1306 
1307 	return __qcom_glink_send(channel, data, len, true);
1308 }
1309 
1310 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1311 {
1312 	struct glink_channel *channel = to_glink_channel(ept);
1313 
1314 	return __qcom_glink_send(channel, data, len, false);
1315 }
1316 
1317 /*
1318  * Finds the device_node for the glink child interested in this channel.
1319  */
1320 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1321 						    const char *channel)
1322 {
1323 	struct device_node *child;
1324 	const char *name;
1325 	const char *key;
1326 	int ret;
1327 
1328 	for_each_available_child_of_node(node, child) {
1329 		key = "qcom,glink-channels";
1330 		ret = of_property_read_string(child, key, &name);
1331 		if (ret)
1332 			continue;
1333 
1334 		if (strcmp(name, channel) == 0)
1335 			return child;
1336 	}
1337 
1338 	return NULL;
1339 }
1340 
1341 static const struct rpmsg_device_ops glink_device_ops = {
1342 	.create_ept = qcom_glink_create_ept,
1343 	.announce_create = qcom_glink_announce_create,
1344 };
1345 
1346 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1347 	.destroy_ept = qcom_glink_destroy_ept,
1348 	.send = qcom_glink_send,
1349 	.trysend = qcom_glink_trysend,
1350 };
1351 
1352 static void qcom_glink_rpdev_release(struct device *dev)
1353 {
1354 	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1355 	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1356 
1357 	channel->rpdev = NULL;
1358 	kfree(rpdev);
1359 }
1360 
1361 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1362 			      char *name)
1363 {
1364 	struct glink_channel *channel;
1365 	struct rpmsg_device *rpdev;
1366 	bool create_device = false;
1367 	struct device_node *node;
1368 	int lcid;
1369 	int ret;
1370 	unsigned long flags;
1371 
1372 	spin_lock_irqsave(&glink->idr_lock, flags);
1373 	idr_for_each_entry(&glink->lcids, channel, lcid) {
1374 		if (!strcmp(channel->name, name))
1375 			break;
1376 	}
1377 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1378 
1379 	if (!channel) {
1380 		channel = qcom_glink_alloc_channel(glink, name);
1381 		if (IS_ERR(channel))
1382 			return PTR_ERR(channel);
1383 
1384 		/* The opening dance was initiated by the remote */
1385 		create_device = true;
1386 	}
1387 
1388 	spin_lock_irqsave(&glink->idr_lock, flags);
1389 	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1390 	if (ret < 0) {
1391 		dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1392 		spin_unlock_irqrestore(&glink->idr_lock, flags);
1393 		goto free_channel;
1394 	}
1395 	channel->rcid = ret;
1396 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1397 
1398 	complete(&channel->open_req);
1399 
1400 	if (create_device) {
1401 		rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1402 		if (!rpdev) {
1403 			ret = -ENOMEM;
1404 			goto rcid_remove;
1405 		}
1406 
1407 		rpdev->ept = &channel->ept;
1408 		strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE);
1409 		rpdev->src = RPMSG_ADDR_ANY;
1410 		rpdev->dst = RPMSG_ADDR_ANY;
1411 		rpdev->ops = &glink_device_ops;
1412 
1413 		node = qcom_glink_match_channel(glink->dev->of_node, name);
1414 		rpdev->dev.of_node = node;
1415 		rpdev->dev.parent = glink->dev;
1416 		rpdev->dev.release = qcom_glink_rpdev_release;
1417 
1418 		ret = rpmsg_register_device(rpdev);
1419 		if (ret)
1420 			goto free_rpdev;
1421 
1422 		channel->rpdev = rpdev;
1423 	}
1424 
1425 	return 0;
1426 
1427 free_rpdev:
1428 	kfree(rpdev);
1429 rcid_remove:
1430 	spin_lock_irqsave(&glink->idr_lock, flags);
1431 	idr_remove(&glink->rcids, channel->rcid);
1432 	channel->rcid = 0;
1433 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1434 free_channel:
1435 	/* Release the reference, iff we took it */
1436 	if (create_device)
1437 		kref_put(&channel->refcount, qcom_glink_channel_release);
1438 
1439 	return ret;
1440 }
1441 
1442 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1443 {
1444 	struct rpmsg_channel_info chinfo;
1445 	struct glink_channel *channel;
1446 	unsigned long flags;
1447 
1448 	spin_lock_irqsave(&glink->idr_lock, flags);
1449 	channel = idr_find(&glink->rcids, rcid);
1450 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1451 	if (WARN(!channel, "close request on unknown channel\n"))
1452 		return;
1453 
1454 	/* cancel pending rx_done work */
1455 	cancel_work_sync(&channel->intent_work);
1456 
1457 	if (channel->rpdev) {
1458 		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
1459 		chinfo.src = RPMSG_ADDR_ANY;
1460 		chinfo.dst = RPMSG_ADDR_ANY;
1461 
1462 		rpmsg_unregister_device(glink->dev, &chinfo);
1463 	}
1464 
1465 	qcom_glink_send_close_ack(glink, channel->rcid);
1466 
1467 	spin_lock_irqsave(&glink->idr_lock, flags);
1468 	idr_remove(&glink->rcids, channel->rcid);
1469 	channel->rcid = 0;
1470 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1471 
1472 	kref_put(&channel->refcount, qcom_glink_channel_release);
1473 }
1474 
1475 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1476 {
1477 	struct glink_channel *channel;
1478 	unsigned long flags;
1479 
1480 	spin_lock_irqsave(&glink->idr_lock, flags);
1481 	channel = idr_find(&glink->lcids, lcid);
1482 	if (WARN(!channel, "close ack on unknown channel\n")) {
1483 		spin_unlock_irqrestore(&glink->idr_lock, flags);
1484 		return;
1485 	}
1486 
1487 	idr_remove(&glink->lcids, channel->lcid);
1488 	channel->lcid = 0;
1489 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1490 
1491 	kref_put(&channel->refcount, qcom_glink_channel_release);
1492 }
1493 
1494 static void qcom_glink_work(struct work_struct *work)
1495 {
1496 	struct qcom_glink *glink = container_of(work, struct qcom_glink,
1497 						rx_work);
1498 	struct glink_defer_cmd *dcmd;
1499 	struct glink_msg *msg;
1500 	unsigned long flags;
1501 	unsigned int param1;
1502 	unsigned int param2;
1503 	unsigned int cmd;
1504 
1505 	for (;;) {
1506 		spin_lock_irqsave(&glink->rx_lock, flags);
1507 		if (list_empty(&glink->rx_queue)) {
1508 			spin_unlock_irqrestore(&glink->rx_lock, flags);
1509 			break;
1510 		}
1511 		dcmd = list_first_entry(&glink->rx_queue,
1512 					struct glink_defer_cmd, node);
1513 		list_del(&dcmd->node);
1514 		spin_unlock_irqrestore(&glink->rx_lock, flags);
1515 
1516 		msg = &dcmd->msg;
1517 		cmd = le16_to_cpu(msg->cmd);
1518 		param1 = le16_to_cpu(msg->param1);
1519 		param2 = le32_to_cpu(msg->param2);
1520 
1521 		switch (cmd) {
1522 		case RPM_CMD_VERSION:
1523 			qcom_glink_receive_version(glink, param1, param2);
1524 			break;
1525 		case RPM_CMD_VERSION_ACK:
1526 			qcom_glink_receive_version_ack(glink, param1, param2);
1527 			break;
1528 		case RPM_CMD_OPEN:
1529 			qcom_glink_rx_open(glink, param1, msg->data);
1530 			break;
1531 		case RPM_CMD_CLOSE:
1532 			qcom_glink_rx_close(glink, param1);
1533 			break;
1534 		case RPM_CMD_CLOSE_ACK:
1535 			qcom_glink_rx_close_ack(glink, param1);
1536 			break;
1537 		case RPM_CMD_RX_INTENT_REQ:
1538 			qcom_glink_handle_intent_req(glink, param1, param2);
1539 			break;
1540 		default:
1541 			WARN(1, "Unknown defer object %d\n", cmd);
1542 			break;
1543 		}
1544 
1545 		kfree(dcmd);
1546 	}
1547 }
1548 
1549 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1550 					   unsigned long features,
1551 					   struct qcom_glink_pipe *rx,
1552 					   struct qcom_glink_pipe *tx,
1553 					   bool intentless)
1554 {
1555 	int irq;
1556 	int ret;
1557 	struct qcom_glink *glink;
1558 
1559 	glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1560 	if (!glink)
1561 		return ERR_PTR(-ENOMEM);
1562 
1563 	glink->dev = dev;
1564 	glink->tx_pipe = tx;
1565 	glink->rx_pipe = rx;
1566 
1567 	glink->features = features;
1568 	glink->intentless = intentless;
1569 
1570 	mutex_init(&glink->tx_lock);
1571 	spin_lock_init(&glink->rx_lock);
1572 	INIT_LIST_HEAD(&glink->rx_queue);
1573 	INIT_WORK(&glink->rx_work, qcom_glink_work);
1574 
1575 	spin_lock_init(&glink->idr_lock);
1576 	idr_init(&glink->lcids);
1577 	idr_init(&glink->rcids);
1578 
1579 	glink->mbox_client.dev = dev;
1580 	glink->mbox_client.knows_txdone = true;
1581 	glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0);
1582 	if (IS_ERR(glink->mbox_chan)) {
1583 		if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER)
1584 			dev_err(dev, "failed to acquire IPC channel\n");
1585 		return ERR_CAST(glink->mbox_chan);
1586 	}
1587 
1588 	irq = of_irq_get(dev->of_node, 0);
1589 	ret = devm_request_irq(dev, irq,
1590 			       qcom_glink_native_intr,
1591 			       IRQF_NO_SUSPEND | IRQF_SHARED,
1592 			       "glink-native", glink);
1593 	if (ret) {
1594 		dev_err(dev, "failed to request IRQ\n");
1595 		return ERR_PTR(ret);
1596 	}
1597 
1598 	glink->irq = irq;
1599 
1600 	ret = qcom_glink_send_version(glink);
1601 	if (ret)
1602 		return ERR_PTR(ret);
1603 
1604 	return glink;
1605 }
1606 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1607 
1608 static int qcom_glink_remove_device(struct device *dev, void *data)
1609 {
1610 	device_unregister(dev);
1611 
1612 	return 0;
1613 }
1614 
1615 void qcom_glink_native_remove(struct qcom_glink *glink)
1616 {
1617 	struct glink_channel *channel;
1618 	int cid;
1619 	int ret;
1620 	unsigned long flags;
1621 
1622 	disable_irq(glink->irq);
1623 	cancel_work_sync(&glink->rx_work);
1624 
1625 	ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1626 	if (ret)
1627 		dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1628 
1629 	spin_lock_irqsave(&glink->idr_lock, flags);
1630 	/* Release any defunct local channels, waiting for close-ack */
1631 	idr_for_each_entry(&glink->lcids, channel, cid)
1632 		kref_put(&channel->refcount, qcom_glink_channel_release);
1633 
1634 	idr_destroy(&glink->lcids);
1635 	idr_destroy(&glink->rcids);
1636 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1637 	mbox_free_channel(glink->mbox_chan);
1638 }
1639 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1640 
1641 void qcom_glink_native_unregister(struct qcom_glink *glink)
1642 {
1643 	device_unregister(glink->dev);
1644 }
1645 EXPORT_SYMBOL_GPL(qcom_glink_native_unregister);
1646 
1647 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1648 MODULE_LICENSE("GPL v2");
1649