1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016-2017, Linaro Ltd
4  */
5 
6 #include <linux/idr.h>
7 #include <linux/interrupt.h>
8 #include <linux/io.h>
9 #include <linux/list.h>
10 #include <linux/mfd/syscon.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/of_address.h>
14 #include <linux/platform_device.h>
15 #include <linux/regmap.h>
16 #include <linux/rpmsg.h>
17 #include <linux/sizes.h>
18 #include <linux/slab.h>
19 #include <linux/wait.h>
20 #include <linux/workqueue.h>
21 #include <linux/mailbox_client.h>
22 
23 #include "rpmsg_internal.h"
24 #include "qcom_glink_native.h"
25 
26 #define GLINK_NAME_SIZE		32
27 #define GLINK_VERSION_1		1
28 
29 #define RPM_GLINK_CID_MIN	1
30 #define RPM_GLINK_CID_MAX	65536
31 
32 struct glink_msg {
33 	__le16 cmd;
34 	__le16 param1;
35 	__le32 param2;
36 	u8 data[];
37 } __packed;
38 
39 /**
40  * struct glink_defer_cmd - deferred incoming control message
41  * @node:	list node
42  * @msg:	message header
43  * @data:	payload of the message
44  *
45  * Copy of a received control message, to be added to @rx_queue and processed
46  * by @rx_work of @qcom_glink.
47  */
48 struct glink_defer_cmd {
49 	struct list_head node;
50 
51 	struct glink_msg msg;
52 	u8 data[];
53 };
54 
55 /**
56  * struct glink_core_rx_intent - RX intent
57  * RX intent
58  *
59  * @data: pointer to the data (may be NULL for zero-copy)
60  * @id: remote or local intent ID
61  * @size: size of the original intent (do not modify)
62  * @reuse: To mark if the intent can be reused after first use
63  * @in_use: To mark if intent is already in use for the channel
64  * @offset: next write offset (initially 0)
65  * @node:	list node
66  */
67 struct glink_core_rx_intent {
68 	void *data;
69 	u32 id;
70 	size_t size;
71 	bool reuse;
72 	bool in_use;
73 	u32 offset;
74 
75 	struct list_head node;
76 };
77 
78 /**
79  * struct qcom_glink - driver context, relates to one remote subsystem
80  * @dev:	reference to the associated struct device
81  * @rx_pipe:	pipe object for receive FIFO
82  * @tx_pipe:	pipe object for transmit FIFO
83  * @rx_work:	worker for handling received control messages
84  * @rx_lock:	protects the @rx_queue
85  * @rx_queue:	queue of received control messages to be processed in @rx_work
86  * @tx_lock:	synchronizes operations on the tx fifo
87  * @idr_lock:	synchronizes @lcids and @rcids modifications
88  * @lcids:	idr of all channels with a known local channel id
89  * @rcids:	idr of all channels with a known remote channel id
90  * @features:	remote features
91  * @intentless:	flag to indicate that there is no intent
92  * @tx_avail_notify: Waitqueue for pending tx tasks
93  * @sent_read_notify: flag to check cmd sent or not
94  * @abort_tx:	flag indicating that all tx attempts should fail
95  */
96 struct qcom_glink {
97 	struct device *dev;
98 
99 	struct qcom_glink_pipe *rx_pipe;
100 	struct qcom_glink_pipe *tx_pipe;
101 
102 	struct work_struct rx_work;
103 	spinlock_t rx_lock;
104 	struct list_head rx_queue;
105 
106 	spinlock_t tx_lock;
107 
108 	spinlock_t idr_lock;
109 	struct idr lcids;
110 	struct idr rcids;
111 	unsigned long features;
112 
113 	bool intentless;
114 	wait_queue_head_t tx_avail_notify;
115 	bool sent_read_notify;
116 
117 	bool abort_tx;
118 };
119 
120 enum {
121 	GLINK_STATE_CLOSED,
122 	GLINK_STATE_OPENING,
123 	GLINK_STATE_OPEN,
124 	GLINK_STATE_CLOSING,
125 };
126 
127 /**
128  * struct glink_channel - internal representation of a channel
129  * @rpdev:	rpdev reference, only used for primary endpoints
130  * @ept:	rpmsg endpoint this channel is associated with
131  * @glink:	qcom_glink context handle
132  * @refcount:	refcount for the channel object
133  * @recv_lock:	guard for @ept.cb
134  * @name:	unique channel name/identifier
135  * @lcid:	channel id, in local space
136  * @rcid:	channel id, in remote space
137  * @intent_lock: lock for protection of @liids, @riids
138  * @liids:	idr of all local intents
139  * @riids:	idr of all remote intents
140  * @intent_work: worker responsible for transmitting rx_done packets
141  * @done_intents: list of intents that needs to be announced rx_done
142  * @buf:	receive buffer, for gathering fragments
143  * @buf_offset:	write offset in @buf
144  * @buf_size:	size of current @buf
145  * @open_ack:	completed once remote has acked the open-request
146  * @open_req:	completed once open-request has been received
147  * @intent_req_lock: Synchronises multiple intent requests
148  * @intent_req_result: Result of intent request
149  * @intent_received: flag indicating that an intent has been received
150  * @intent_req_wq: wait queue for intent_req signalling
151  */
152 struct glink_channel {
153 	struct rpmsg_endpoint ept;
154 
155 	struct rpmsg_device *rpdev;
156 	struct qcom_glink *glink;
157 
158 	struct kref refcount;
159 
160 	spinlock_t recv_lock;
161 
162 	char *name;
163 	unsigned int lcid;
164 	unsigned int rcid;
165 
166 	spinlock_t intent_lock;
167 	struct idr liids;
168 	struct idr riids;
169 	struct work_struct intent_work;
170 	struct list_head done_intents;
171 
172 	struct glink_core_rx_intent *buf;
173 	int buf_offset;
174 	int buf_size;
175 
176 	struct completion open_ack;
177 	struct completion open_req;
178 
179 	struct mutex intent_req_lock;
180 	int intent_req_result;
181 	bool intent_received;
182 	wait_queue_head_t intent_req_wq;
183 };
184 
185 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
186 
187 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
188 
189 #define GLINK_CMD_VERSION		0
190 #define GLINK_CMD_VERSION_ACK		1
191 #define GLINK_CMD_OPEN			2
192 #define GLINK_CMD_CLOSE			3
193 #define GLINK_CMD_OPEN_ACK		4
194 #define GLINK_CMD_INTENT		5
195 #define GLINK_CMD_RX_DONE		6
196 #define GLINK_CMD_RX_INTENT_REQ		7
197 #define GLINK_CMD_RX_INTENT_REQ_ACK	8
198 #define GLINK_CMD_TX_DATA		9
199 #define GLINK_CMD_CLOSE_ACK		11
200 #define GLINK_CMD_TX_DATA_CONT		12
201 #define GLINK_CMD_READ_NOTIF		13
202 #define GLINK_CMD_RX_DONE_W_REUSE	14
203 #define GLINK_CMD_SIGNALS		15
204 
205 #define GLINK_FEATURE_INTENTLESS	BIT(1)
206 
207 #define NATIVE_DTR_SIG			NATIVE_DSR_SIG
208 #define NATIVE_DSR_SIG			BIT(31)
209 #define NATIVE_RTS_SIG			NATIVE_CTS_SIG
210 #define NATIVE_CTS_SIG			BIT(30)
211 
212 static void qcom_glink_rx_done_work(struct work_struct *work);
213 
214 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
215 						      const char *name)
216 {
217 	struct glink_channel *channel;
218 
219 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
220 	if (!channel)
221 		return ERR_PTR(-ENOMEM);
222 
223 	/* Setup glink internal glink_channel data */
224 	spin_lock_init(&channel->recv_lock);
225 	spin_lock_init(&channel->intent_lock);
226 	mutex_init(&channel->intent_req_lock);
227 
228 	channel->glink = glink;
229 	channel->name = kstrdup(name, GFP_KERNEL);
230 	if (!channel->name) {
231 		kfree(channel);
232 		return ERR_PTR(-ENOMEM);
233 	}
234 
235 	init_completion(&channel->open_req);
236 	init_completion(&channel->open_ack);
237 	init_waitqueue_head(&channel->intent_req_wq);
238 
239 	INIT_LIST_HEAD(&channel->done_intents);
240 	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
241 
242 	idr_init(&channel->liids);
243 	idr_init(&channel->riids);
244 	kref_init(&channel->refcount);
245 
246 	return channel;
247 }
248 
249 static void qcom_glink_channel_release(struct kref *ref)
250 {
251 	struct glink_channel *channel = container_of(ref, struct glink_channel,
252 						     refcount);
253 	struct glink_core_rx_intent *intent;
254 	struct glink_core_rx_intent *tmp;
255 	unsigned long flags;
256 	int iid;
257 
258 	/* cancel pending rx_done work */
259 	cancel_work_sync(&channel->intent_work);
260 
261 	spin_lock_irqsave(&channel->intent_lock, flags);
262 	/* Free all non-reuse intents pending rx_done work */
263 	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
264 		if (!intent->reuse) {
265 			kfree(intent->data);
266 			kfree(intent);
267 		}
268 	}
269 
270 	idr_for_each_entry(&channel->liids, tmp, iid) {
271 		kfree(tmp->data);
272 		kfree(tmp);
273 	}
274 	idr_destroy(&channel->liids);
275 
276 	idr_for_each_entry(&channel->riids, tmp, iid)
277 		kfree(tmp);
278 	idr_destroy(&channel->riids);
279 	spin_unlock_irqrestore(&channel->intent_lock, flags);
280 
281 	kfree(channel->name);
282 	kfree(channel);
283 }
284 
285 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
286 {
287 	return glink->rx_pipe->avail(glink->rx_pipe);
288 }
289 
290 static void qcom_glink_rx_peek(struct qcom_glink *glink,
291 			       void *data, unsigned int offset, size_t count)
292 {
293 	glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
294 }
295 
296 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
297 {
298 	glink->rx_pipe->advance(glink->rx_pipe, count);
299 }
300 
301 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
302 {
303 	return glink->tx_pipe->avail(glink->tx_pipe);
304 }
305 
306 static void qcom_glink_tx_write(struct qcom_glink *glink,
307 				const void *hdr, size_t hlen,
308 				const void *data, size_t dlen)
309 {
310 	glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
311 }
312 
313 static void qcom_glink_tx_kick(struct qcom_glink *glink)
314 {
315 	glink->tx_pipe->kick(glink->tx_pipe);
316 }
317 
318 static void qcom_glink_send_read_notify(struct qcom_glink *glink)
319 {
320 	struct glink_msg msg;
321 
322 	msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
323 	msg.param1 = 0;
324 	msg.param2 = 0;
325 
326 	qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
327 
328 	qcom_glink_tx_kick(glink);
329 }
330 
331 static int qcom_glink_tx(struct qcom_glink *glink,
332 			 const void *hdr, size_t hlen,
333 			 const void *data, size_t dlen, bool wait)
334 {
335 	unsigned int tlen = hlen + dlen;
336 	unsigned long flags;
337 	int ret = 0;
338 
339 	/* Reject packets that are too big */
340 	if (tlen >= glink->tx_pipe->length)
341 		return -EINVAL;
342 
343 	spin_lock_irqsave(&glink->tx_lock, flags);
344 
345 	if (glink->abort_tx) {
346 		ret = -EIO;
347 		goto out;
348 	}
349 
350 	while (qcom_glink_tx_avail(glink) < tlen) {
351 		if (!wait) {
352 			ret = -EAGAIN;
353 			goto out;
354 		}
355 
356 		if (glink->abort_tx) {
357 			ret = -EIO;
358 			goto out;
359 		}
360 
361 		if (!glink->sent_read_notify) {
362 			glink->sent_read_notify = true;
363 			qcom_glink_send_read_notify(glink);
364 		}
365 
366 		/* Wait without holding the tx_lock */
367 		spin_unlock_irqrestore(&glink->tx_lock, flags);
368 
369 		wait_event_timeout(glink->tx_avail_notify,
370 				   qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
371 
372 		spin_lock_irqsave(&glink->tx_lock, flags);
373 
374 		if (qcom_glink_tx_avail(glink) >= tlen)
375 			glink->sent_read_notify = false;
376 	}
377 
378 	qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
379 	qcom_glink_tx_kick(glink);
380 
381 out:
382 	spin_unlock_irqrestore(&glink->tx_lock, flags);
383 
384 	return ret;
385 }
386 
387 static int qcom_glink_send_version(struct qcom_glink *glink)
388 {
389 	struct glink_msg msg;
390 
391 	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
392 	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
393 	msg.param2 = cpu_to_le32(glink->features);
394 
395 	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
396 }
397 
398 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
399 {
400 	struct glink_msg msg;
401 
402 	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
403 	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
404 	msg.param2 = cpu_to_le32(glink->features);
405 
406 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
407 }
408 
409 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
410 				     struct glink_channel *channel)
411 {
412 	struct glink_msg msg;
413 
414 	msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
415 	msg.param1 = cpu_to_le16(channel->rcid);
416 	msg.param2 = cpu_to_le32(0);
417 
418 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
419 }
420 
421 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
422 					     unsigned int cid, bool granted)
423 {
424 	struct glink_channel *channel;
425 	unsigned long flags;
426 
427 	spin_lock_irqsave(&glink->idr_lock, flags);
428 	channel = idr_find(&glink->rcids, cid);
429 	spin_unlock_irqrestore(&glink->idr_lock, flags);
430 	if (!channel) {
431 		dev_err(glink->dev, "unable to find channel\n");
432 		return;
433 	}
434 
435 	WRITE_ONCE(channel->intent_req_result, granted);
436 	wake_up_all(&channel->intent_req_wq);
437 }
438 
439 static void qcom_glink_intent_req_abort(struct glink_channel *channel)
440 {
441 	WRITE_ONCE(channel->intent_req_result, 0);
442 	wake_up_all(&channel->intent_req_wq);
443 }
444 
445 /**
446  * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
447  * @glink: Ptr to the glink edge
448  * @channel: Ptr to the channel that the open req is sent
449  *
450  * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
451  * Will return with refcount held, regardless of outcome.
452  *
453  * Return: 0 on success, negative errno otherwise.
454  */
455 static int qcom_glink_send_open_req(struct qcom_glink *glink,
456 				    struct glink_channel *channel)
457 {
458 	struct {
459 		struct glink_msg msg;
460 		u8 name[GLINK_NAME_SIZE];
461 	} __packed req;
462 	int name_len = strlen(channel->name) + 1;
463 	int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
464 	int ret;
465 	unsigned long flags;
466 
467 	kref_get(&channel->refcount);
468 
469 	spin_lock_irqsave(&glink->idr_lock, flags);
470 	ret = idr_alloc_cyclic(&glink->lcids, channel,
471 			       RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
472 			       GFP_ATOMIC);
473 	spin_unlock_irqrestore(&glink->idr_lock, flags);
474 	if (ret < 0)
475 		return ret;
476 
477 	channel->lcid = ret;
478 
479 	req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN);
480 	req.msg.param1 = cpu_to_le16(channel->lcid);
481 	req.msg.param2 = cpu_to_le32(name_len);
482 	strcpy(req.name, channel->name);
483 
484 	ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
485 	if (ret)
486 		goto remove_idr;
487 
488 	return 0;
489 
490 remove_idr:
491 	spin_lock_irqsave(&glink->idr_lock, flags);
492 	idr_remove(&glink->lcids, channel->lcid);
493 	channel->lcid = 0;
494 	spin_unlock_irqrestore(&glink->idr_lock, flags);
495 
496 	return ret;
497 }
498 
499 static void qcom_glink_send_close_req(struct qcom_glink *glink,
500 				      struct glink_channel *channel)
501 {
502 	struct glink_msg req;
503 
504 	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
505 	req.param1 = cpu_to_le16(channel->lcid);
506 	req.param2 = 0;
507 
508 	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
509 }
510 
511 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
512 				      unsigned int rcid)
513 {
514 	struct glink_msg req;
515 
516 	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
517 	req.param1 = cpu_to_le16(rcid);
518 	req.param2 = 0;
519 
520 	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
521 }
522 
523 static void qcom_glink_rx_done_work(struct work_struct *work)
524 {
525 	struct glink_channel *channel = container_of(work, struct glink_channel,
526 						     intent_work);
527 	struct qcom_glink *glink = channel->glink;
528 	struct glink_core_rx_intent *intent, *tmp;
529 	struct {
530 		u16 id;
531 		u16 lcid;
532 		u32 liid;
533 	} __packed cmd;
534 
535 	unsigned int cid = channel->lcid;
536 	unsigned int iid;
537 	bool reuse;
538 	unsigned long flags;
539 
540 	spin_lock_irqsave(&channel->intent_lock, flags);
541 	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
542 		list_del(&intent->node);
543 		spin_unlock_irqrestore(&channel->intent_lock, flags);
544 		iid = intent->id;
545 		reuse = intent->reuse;
546 
547 		cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
548 		cmd.lcid = cid;
549 		cmd.liid = iid;
550 
551 		qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
552 		if (!reuse) {
553 			kfree(intent->data);
554 			kfree(intent);
555 		}
556 		spin_lock_irqsave(&channel->intent_lock, flags);
557 	}
558 	spin_unlock_irqrestore(&channel->intent_lock, flags);
559 }
560 
561 static void qcom_glink_rx_done(struct qcom_glink *glink,
562 			       struct glink_channel *channel,
563 			       struct glink_core_rx_intent *intent)
564 {
565 	/* We don't send RX_DONE to intentless systems */
566 	if (glink->intentless) {
567 		kfree(intent->data);
568 		kfree(intent);
569 		return;
570 	}
571 
572 	/* Take it off the tree of receive intents */
573 	if (!intent->reuse) {
574 		spin_lock(&channel->intent_lock);
575 		idr_remove(&channel->liids, intent->id);
576 		spin_unlock(&channel->intent_lock);
577 	}
578 
579 	/* Schedule the sending of a rx_done indication */
580 	spin_lock(&channel->intent_lock);
581 	list_add_tail(&intent->node, &channel->done_intents);
582 	spin_unlock(&channel->intent_lock);
583 
584 	schedule_work(&channel->intent_work);
585 }
586 
587 /**
588  * qcom_glink_receive_version() - receive version/features from remote system
589  *
590  * @glink:	pointer to transport interface
591  * @version:	remote version
592  * @features:	remote features
593  *
594  * This function is called in response to a remote-initiated version/feature
595  * negotiation sequence.
596  */
597 static void qcom_glink_receive_version(struct qcom_glink *glink,
598 				       u32 version,
599 				       u32 features)
600 {
601 	switch (version) {
602 	case 0:
603 		break;
604 	case GLINK_VERSION_1:
605 		glink->features &= features;
606 		fallthrough;
607 	default:
608 		qcom_glink_send_version_ack(glink);
609 		break;
610 	}
611 }
612 
613 /**
614  * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
615  *
616  * @glink:	pointer to transport interface
617  * @version:	remote version response
618  * @features:	remote features response
619  *
620  * This function is called in response to a local-initiated version/feature
621  * negotiation sequence and is the counter-offer from the remote side based
622  * upon the initial version and feature set requested.
623  */
624 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
625 					   u32 version,
626 					   u32 features)
627 {
628 	switch (version) {
629 	case 0:
630 		/* Version negotiation failed */
631 		break;
632 	case GLINK_VERSION_1:
633 		if (features == glink->features)
634 			break;
635 
636 		glink->features &= features;
637 		fallthrough;
638 	default:
639 		qcom_glink_send_version(glink);
640 		break;
641 	}
642 }
643 
644 /**
645  * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
646  * 	wire format and transmit
647  * @glink:	The transport to transmit on.
648  * @channel:	The glink channel
649  * @granted:	The request response to encode.
650  *
651  * Return: 0 on success or standard Linux error code.
652  */
653 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
654 					  struct glink_channel *channel,
655 					  bool granted)
656 {
657 	struct glink_msg msg;
658 
659 	msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
660 	msg.param1 = cpu_to_le16(channel->lcid);
661 	msg.param2 = cpu_to_le32(granted);
662 
663 	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
664 
665 	return 0;
666 }
667 
668 /**
669  * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
670  *			   transmit
671  * @glink:	The transport to transmit on.
672  * @channel:	The local channel
673  * @intent:	The intent to pass on to remote.
674  *
675  * Return: 0 on success or standard Linux error code.
676  */
677 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
678 				       struct glink_channel *channel,
679 				       struct glink_core_rx_intent *intent)
680 {
681 	struct command {
682 		__le16 id;
683 		__le16 lcid;
684 		__le32 count;
685 		__le32 size;
686 		__le32 liid;
687 	} __packed;
688 	struct command cmd;
689 
690 	cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
691 	cmd.lcid = cpu_to_le16(channel->lcid);
692 	cmd.count = cpu_to_le32(1);
693 	cmd.size = cpu_to_le32(intent->size);
694 	cmd.liid = cpu_to_le32(intent->id);
695 
696 	qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
697 
698 	return 0;
699 }
700 
701 static struct glink_core_rx_intent *
702 qcom_glink_alloc_intent(struct qcom_glink *glink,
703 			struct glink_channel *channel,
704 			size_t size,
705 			bool reuseable)
706 {
707 	struct glink_core_rx_intent *intent;
708 	int ret;
709 	unsigned long flags;
710 
711 	intent = kzalloc(sizeof(*intent), GFP_KERNEL);
712 	if (!intent)
713 		return NULL;
714 
715 	intent->data = kzalloc(size, GFP_KERNEL);
716 	if (!intent->data)
717 		goto free_intent;
718 
719 	spin_lock_irqsave(&channel->intent_lock, flags);
720 	ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
721 	if (ret < 0) {
722 		spin_unlock_irqrestore(&channel->intent_lock, flags);
723 		goto free_data;
724 	}
725 	spin_unlock_irqrestore(&channel->intent_lock, flags);
726 
727 	intent->id = ret;
728 	intent->size = size;
729 	intent->reuse = reuseable;
730 
731 	return intent;
732 
733 free_data:
734 	kfree(intent->data);
735 free_intent:
736 	kfree(intent);
737 	return NULL;
738 }
739 
740 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
741 				      u32 cid, uint32_t iid,
742 				      bool reuse)
743 {
744 	struct glink_core_rx_intent *intent;
745 	struct glink_channel *channel;
746 	unsigned long flags;
747 
748 	spin_lock_irqsave(&glink->idr_lock, flags);
749 	channel = idr_find(&glink->rcids, cid);
750 	spin_unlock_irqrestore(&glink->idr_lock, flags);
751 	if (!channel) {
752 		dev_err(glink->dev, "invalid channel id received\n");
753 		return;
754 	}
755 
756 	spin_lock_irqsave(&channel->intent_lock, flags);
757 	intent = idr_find(&channel->riids, iid);
758 
759 	if (!intent) {
760 		spin_unlock_irqrestore(&channel->intent_lock, flags);
761 		dev_err(glink->dev, "invalid intent id received\n");
762 		return;
763 	}
764 
765 	intent->in_use = false;
766 
767 	if (!reuse) {
768 		idr_remove(&channel->riids, intent->id);
769 		kfree(intent);
770 	}
771 	spin_unlock_irqrestore(&channel->intent_lock, flags);
772 
773 	if (reuse) {
774 		WRITE_ONCE(channel->intent_received, true);
775 		wake_up_all(&channel->intent_req_wq);
776 	}
777 }
778 
779 /**
780  * qcom_glink_handle_intent_req() - Receive a request for rx_intent
781  *					    from remote side
782  * @glink:      Pointer to the transport interface
783  * @cid:	Remote channel ID
784  * @size:	size of the intent
785  *
786  * The function searches for the local channel to which the request for
787  * rx_intent has arrived and allocates and notifies the remote back
788  */
789 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
790 					 u32 cid, size_t size)
791 {
792 	struct glink_core_rx_intent *intent;
793 	struct glink_channel *channel;
794 	unsigned long flags;
795 
796 	spin_lock_irqsave(&glink->idr_lock, flags);
797 	channel = idr_find(&glink->rcids, cid);
798 	spin_unlock_irqrestore(&glink->idr_lock, flags);
799 
800 	if (!channel) {
801 		pr_err("%s channel not found for cid %d\n", __func__, cid);
802 		return;
803 	}
804 
805 	intent = qcom_glink_alloc_intent(glink, channel, size, false);
806 	if (intent)
807 		qcom_glink_advertise_intent(glink, channel, intent);
808 
809 	qcom_glink_send_intent_req_ack(glink, channel, !!intent);
810 }
811 
812 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
813 {
814 	struct glink_defer_cmd *dcmd;
815 
816 	extra = ALIGN(extra, 8);
817 
818 	if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
819 		dev_dbg(glink->dev, "Insufficient data in rx fifo");
820 		return -ENXIO;
821 	}
822 
823 	dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
824 	if (!dcmd)
825 		return -ENOMEM;
826 
827 	INIT_LIST_HEAD(&dcmd->node);
828 
829 	qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
830 
831 	spin_lock(&glink->rx_lock);
832 	list_add_tail(&dcmd->node, &glink->rx_queue);
833 	spin_unlock(&glink->rx_lock);
834 
835 	schedule_work(&glink->rx_work);
836 	qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
837 
838 	return 0;
839 }
840 
841 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
842 {
843 	struct glink_core_rx_intent *intent;
844 	struct glink_channel *channel;
845 	struct {
846 		struct glink_msg msg;
847 		__le32 chunk_size;
848 		__le32 left_size;
849 	} __packed hdr;
850 	unsigned int chunk_size;
851 	unsigned int left_size;
852 	unsigned int rcid;
853 	unsigned int liid;
854 	int ret = 0;
855 	unsigned long flags;
856 
857 	if (avail < sizeof(hdr)) {
858 		dev_dbg(glink->dev, "Not enough data in fifo\n");
859 		return -EAGAIN;
860 	}
861 
862 	qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr));
863 	chunk_size = le32_to_cpu(hdr.chunk_size);
864 	left_size = le32_to_cpu(hdr.left_size);
865 
866 	if (avail < sizeof(hdr) + chunk_size) {
867 		dev_dbg(glink->dev, "Payload not yet in fifo\n");
868 		return -EAGAIN;
869 	}
870 
871 	rcid = le16_to_cpu(hdr.msg.param1);
872 	spin_lock_irqsave(&glink->idr_lock, flags);
873 	channel = idr_find(&glink->rcids, rcid);
874 	spin_unlock_irqrestore(&glink->idr_lock, flags);
875 	if (!channel) {
876 		dev_dbg(glink->dev, "Data on non-existing channel\n");
877 
878 		/* Drop the message */
879 		goto advance_rx;
880 	}
881 
882 	if (glink->intentless) {
883 		/* Might have an ongoing, fragmented, message to append */
884 		if (!channel->buf) {
885 			intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
886 			if (!intent)
887 				return -ENOMEM;
888 
889 			intent->data = kmalloc(chunk_size + left_size,
890 					       GFP_ATOMIC);
891 			if (!intent->data) {
892 				kfree(intent);
893 				return -ENOMEM;
894 			}
895 
896 			intent->id = 0xbabababa;
897 			intent->size = chunk_size + left_size;
898 			intent->offset = 0;
899 
900 			channel->buf = intent;
901 		} else {
902 			intent = channel->buf;
903 		}
904 	} else {
905 		liid = le32_to_cpu(hdr.msg.param2);
906 
907 		spin_lock_irqsave(&channel->intent_lock, flags);
908 		intent = idr_find(&channel->liids, liid);
909 		spin_unlock_irqrestore(&channel->intent_lock, flags);
910 
911 		if (!intent) {
912 			dev_err(glink->dev,
913 				"no intent found for channel %s intent %d",
914 				channel->name, liid);
915 			ret = -ENOENT;
916 			goto advance_rx;
917 		}
918 	}
919 
920 	if (intent->size - intent->offset < chunk_size) {
921 		dev_err(glink->dev, "Insufficient space in intent\n");
922 
923 		/* The packet header lied, drop payload */
924 		goto advance_rx;
925 	}
926 
927 	qcom_glink_rx_peek(glink, intent->data + intent->offset,
928 			   sizeof(hdr), chunk_size);
929 	intent->offset += chunk_size;
930 
931 	/* Handle message when no fragments remain to be received */
932 	if (!left_size) {
933 		spin_lock(&channel->recv_lock);
934 		if (channel->ept.cb) {
935 			channel->ept.cb(channel->ept.rpdev,
936 					intent->data,
937 					intent->offset,
938 					channel->ept.priv,
939 					RPMSG_ADDR_ANY);
940 		}
941 		spin_unlock(&channel->recv_lock);
942 
943 		intent->offset = 0;
944 		channel->buf = NULL;
945 
946 		qcom_glink_rx_done(glink, channel, intent);
947 	}
948 
949 advance_rx:
950 	qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
951 
952 	return ret;
953 }
954 
955 static void qcom_glink_handle_intent(struct qcom_glink *glink,
956 				     unsigned int cid,
957 				     unsigned int count,
958 				     size_t avail)
959 {
960 	struct glink_core_rx_intent *intent;
961 	struct glink_channel *channel;
962 	struct intent_pair {
963 		__le32 size;
964 		__le32 iid;
965 	};
966 
967 	struct {
968 		struct glink_msg msg;
969 		struct intent_pair intents[];
970 	} __packed * msg;
971 
972 	const size_t msglen = struct_size(msg, intents, count);
973 	int ret;
974 	int i;
975 	unsigned long flags;
976 
977 	if (avail < msglen) {
978 		dev_dbg(glink->dev, "Not enough data in fifo\n");
979 		return;
980 	}
981 
982 	spin_lock_irqsave(&glink->idr_lock, flags);
983 	channel = idr_find(&glink->rcids, cid);
984 	spin_unlock_irqrestore(&glink->idr_lock, flags);
985 	if (!channel) {
986 		dev_err(glink->dev, "intents for non-existing channel\n");
987 		qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
988 		return;
989 	}
990 
991 	msg = kmalloc(msglen, GFP_ATOMIC);
992 	if (!msg)
993 		return;
994 
995 	qcom_glink_rx_peek(glink, msg, 0, msglen);
996 
997 	for (i = 0; i < count; ++i) {
998 		intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
999 		if (!intent)
1000 			break;
1001 
1002 		intent->id = le32_to_cpu(msg->intents[i].iid);
1003 		intent->size = le32_to_cpu(msg->intents[i].size);
1004 
1005 		spin_lock_irqsave(&channel->intent_lock, flags);
1006 		ret = idr_alloc(&channel->riids, intent,
1007 				intent->id, intent->id + 1, GFP_ATOMIC);
1008 		spin_unlock_irqrestore(&channel->intent_lock, flags);
1009 
1010 		if (ret < 0)
1011 			dev_err(glink->dev, "failed to store remote intent\n");
1012 	}
1013 
1014 	WRITE_ONCE(channel->intent_received, true);
1015 	wake_up_all(&channel->intent_req_wq);
1016 
1017 	kfree(msg);
1018 	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1019 }
1020 
1021 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1022 {
1023 	struct glink_channel *channel;
1024 
1025 	spin_lock(&glink->idr_lock);
1026 	channel = idr_find(&glink->lcids, lcid);
1027 	spin_unlock(&glink->idr_lock);
1028 	if (!channel) {
1029 		dev_err(glink->dev, "Invalid open ack packet\n");
1030 		return -EINVAL;
1031 	}
1032 
1033 	complete_all(&channel->open_ack);
1034 
1035 	return 0;
1036 }
1037 
1038 /**
1039  * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1040  * @ept:	Rpmsg endpoint for channel.
1041  * @pause:	Pause transmission
1042  * @dst:	destination address of the endpoint
1043  *
1044  * Return: 0 on success or standard Linux error code.
1045  */
1046 static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1047 {
1048 	struct glink_channel *channel = to_glink_channel(ept);
1049 	struct qcom_glink *glink = channel->glink;
1050 	struct glink_msg msg;
1051 	u32 sigs = 0;
1052 
1053 	if (pause)
1054 		sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1055 
1056 	msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1057 	msg.param1 = cpu_to_le16(channel->lcid);
1058 	msg.param2 = cpu_to_le32(sigs);
1059 
1060 	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
1061 }
1062 
1063 static void qcom_glink_handle_signals(struct qcom_glink *glink,
1064 				      unsigned int rcid, unsigned int sigs)
1065 {
1066 	struct glink_channel *channel;
1067 	unsigned long flags;
1068 	bool enable;
1069 
1070 	spin_lock_irqsave(&glink->idr_lock, flags);
1071 	channel = idr_find(&glink->rcids, rcid);
1072 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1073 	if (!channel) {
1074 		dev_err(glink->dev, "signal for non-existing channel\n");
1075 		return;
1076 	}
1077 
1078 	enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1079 
1080 	if (channel->ept.flow_cb)
1081 		channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1082 }
1083 
1084 void qcom_glink_native_rx(struct qcom_glink *glink)
1085 {
1086 	struct glink_msg msg;
1087 	unsigned int param1;
1088 	unsigned int param2;
1089 	unsigned int avail;
1090 	unsigned int cmd;
1091 	int ret = 0;
1092 
1093 	/* To wakeup any blocking writers */
1094 	wake_up_all(&glink->tx_avail_notify);
1095 
1096 	for (;;) {
1097 		avail = qcom_glink_rx_avail(glink);
1098 		if (avail < sizeof(msg))
1099 			break;
1100 
1101 		qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg));
1102 
1103 		cmd = le16_to_cpu(msg.cmd);
1104 		param1 = le16_to_cpu(msg.param1);
1105 		param2 = le32_to_cpu(msg.param2);
1106 
1107 		switch (cmd) {
1108 		case GLINK_CMD_VERSION:
1109 		case GLINK_CMD_VERSION_ACK:
1110 		case GLINK_CMD_CLOSE:
1111 		case GLINK_CMD_CLOSE_ACK:
1112 		case GLINK_CMD_RX_INTENT_REQ:
1113 			ret = qcom_glink_rx_defer(glink, 0);
1114 			break;
1115 		case GLINK_CMD_OPEN_ACK:
1116 			ret = qcom_glink_rx_open_ack(glink, param1);
1117 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1118 			break;
1119 		case GLINK_CMD_OPEN:
1120 			ret = qcom_glink_rx_defer(glink, param2);
1121 			break;
1122 		case GLINK_CMD_TX_DATA:
1123 		case GLINK_CMD_TX_DATA_CONT:
1124 			ret = qcom_glink_rx_data(glink, avail);
1125 			break;
1126 		case GLINK_CMD_READ_NOTIF:
1127 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1128 			qcom_glink_tx_kick(glink);
1129 			break;
1130 		case GLINK_CMD_INTENT:
1131 			qcom_glink_handle_intent(glink, param1, param2, avail);
1132 			break;
1133 		case GLINK_CMD_RX_DONE:
1134 			qcom_glink_handle_rx_done(glink, param1, param2, false);
1135 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1136 			break;
1137 		case GLINK_CMD_RX_DONE_W_REUSE:
1138 			qcom_glink_handle_rx_done(glink, param1, param2, true);
1139 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1140 			break;
1141 		case GLINK_CMD_RX_INTENT_REQ_ACK:
1142 			qcom_glink_handle_intent_req_ack(glink, param1, param2);
1143 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1144 			break;
1145 		case GLINK_CMD_SIGNALS:
1146 			qcom_glink_handle_signals(glink, param1, param2);
1147 			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1148 			break;
1149 		default:
1150 			dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1151 			ret = -EINVAL;
1152 			break;
1153 		}
1154 
1155 		if (ret)
1156 			break;
1157 	}
1158 }
1159 EXPORT_SYMBOL(qcom_glink_native_rx);
1160 
1161 /* Locally initiated rpmsg_create_ept */
1162 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1163 						     const char *name)
1164 {
1165 	struct glink_channel *channel;
1166 	int ret;
1167 	unsigned long flags;
1168 
1169 	channel = qcom_glink_alloc_channel(glink, name);
1170 	if (IS_ERR(channel))
1171 		return ERR_CAST(channel);
1172 
1173 	ret = qcom_glink_send_open_req(glink, channel);
1174 	if (ret)
1175 		goto release_channel;
1176 
1177 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1178 	if (!ret)
1179 		goto err_timeout;
1180 
1181 	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1182 	if (!ret)
1183 		goto err_timeout;
1184 
1185 	qcom_glink_send_open_ack(glink, channel);
1186 
1187 	return channel;
1188 
1189 err_timeout:
1190 	/* qcom_glink_send_open_req() did register the channel in lcids*/
1191 	spin_lock_irqsave(&glink->idr_lock, flags);
1192 	idr_remove(&glink->lcids, channel->lcid);
1193 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1194 
1195 release_channel:
1196 	/* Release qcom_glink_send_open_req() reference */
1197 	kref_put(&channel->refcount, qcom_glink_channel_release);
1198 	/* Release qcom_glink_alloc_channel() reference */
1199 	kref_put(&channel->refcount, qcom_glink_channel_release);
1200 
1201 	return ERR_PTR(-ETIMEDOUT);
1202 }
1203 
1204 /* Remote initiated rpmsg_create_ept */
1205 static int qcom_glink_create_remote(struct qcom_glink *glink,
1206 				    struct glink_channel *channel)
1207 {
1208 	int ret;
1209 
1210 	qcom_glink_send_open_ack(glink, channel);
1211 
1212 	ret = qcom_glink_send_open_req(glink, channel);
1213 	if (ret)
1214 		goto close_link;
1215 
1216 	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1217 	if (!ret) {
1218 		ret = -ETIMEDOUT;
1219 		goto close_link;
1220 	}
1221 
1222 	return 0;
1223 
1224 close_link:
1225 	/*
1226 	 * Send a close request to "undo" our open-ack. The close-ack will
1227 	 * release qcom_glink_send_open_req() reference and the last reference
1228 	 * will be relesed after receiving remote_close or transport unregister
1229 	 * by calling qcom_glink_native_remove().
1230 	 */
1231 	qcom_glink_send_close_req(glink, channel);
1232 
1233 	return ret;
1234 }
1235 
1236 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1237 						    rpmsg_rx_cb_t cb,
1238 						    void *priv,
1239 						    struct rpmsg_channel_info
1240 									chinfo)
1241 {
1242 	struct glink_channel *parent = to_glink_channel(rpdev->ept);
1243 	struct glink_channel *channel;
1244 	struct qcom_glink *glink = parent->glink;
1245 	struct rpmsg_endpoint *ept;
1246 	const char *name = chinfo.name;
1247 	int cid;
1248 	int ret;
1249 	unsigned long flags;
1250 
1251 	spin_lock_irqsave(&glink->idr_lock, flags);
1252 	idr_for_each_entry(&glink->rcids, channel, cid) {
1253 		if (!strcmp(channel->name, name))
1254 			break;
1255 	}
1256 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1257 
1258 	if (!channel) {
1259 		channel = qcom_glink_create_local(glink, name);
1260 		if (IS_ERR(channel))
1261 			return NULL;
1262 	} else {
1263 		ret = qcom_glink_create_remote(glink, channel);
1264 		if (ret)
1265 			return NULL;
1266 	}
1267 
1268 	ept = &channel->ept;
1269 	ept->rpdev = rpdev;
1270 	ept->cb = cb;
1271 	ept->priv = priv;
1272 	ept->ops = &glink_endpoint_ops;
1273 
1274 	return ept;
1275 }
1276 
1277 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1278 {
1279 	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1280 	struct device_node *np = rpdev->dev.of_node;
1281 	struct qcom_glink *glink = channel->glink;
1282 	struct glink_core_rx_intent *intent;
1283 	const struct property *prop = NULL;
1284 	__be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1285 	int num_intents;
1286 	int num_groups = 1;
1287 	__be32 *val = defaults;
1288 	int size;
1289 
1290 	if (glink->intentless || !completion_done(&channel->open_ack))
1291 		return 0;
1292 
1293 	prop = of_find_property(np, "qcom,intents", NULL);
1294 	if (prop) {
1295 		val = prop->value;
1296 		num_groups = prop->length / sizeof(u32) / 2;
1297 	}
1298 
1299 	/* Channel is now open, advertise base set of intents */
1300 	while (num_groups--) {
1301 		size = be32_to_cpup(val++);
1302 		num_intents = be32_to_cpup(val++);
1303 		while (num_intents--) {
1304 			intent = qcom_glink_alloc_intent(glink, channel, size,
1305 							 true);
1306 			if (!intent)
1307 				break;
1308 
1309 			qcom_glink_advertise_intent(glink, channel, intent);
1310 		}
1311 	}
1312 	return 0;
1313 }
1314 
1315 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1316 {
1317 	struct glink_channel *channel = to_glink_channel(ept);
1318 	struct qcom_glink *glink = channel->glink;
1319 	unsigned long flags;
1320 
1321 	spin_lock_irqsave(&channel->recv_lock, flags);
1322 	channel->ept.cb = NULL;
1323 	spin_unlock_irqrestore(&channel->recv_lock, flags);
1324 
1325 	/* Decouple the potential rpdev from the channel */
1326 	channel->rpdev = NULL;
1327 
1328 	qcom_glink_send_close_req(glink, channel);
1329 }
1330 
1331 static int qcom_glink_request_intent(struct qcom_glink *glink,
1332 				     struct glink_channel *channel,
1333 				     size_t size)
1334 {
1335 	struct {
1336 		u16 id;
1337 		u16 cid;
1338 		u32 size;
1339 	} __packed cmd;
1340 
1341 	int ret;
1342 
1343 	mutex_lock(&channel->intent_req_lock);
1344 
1345 	WRITE_ONCE(channel->intent_req_result, -1);
1346 	WRITE_ONCE(channel->intent_received, false);
1347 
1348 	cmd.id = GLINK_CMD_RX_INTENT_REQ;
1349 	cmd.cid = channel->lcid;
1350 	cmd.size = size;
1351 
1352 	ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1353 	if (ret)
1354 		goto unlock;
1355 
1356 	ret = wait_event_timeout(channel->intent_req_wq,
1357 				 READ_ONCE(channel->intent_req_result) >= 0 &&
1358 				 READ_ONCE(channel->intent_received),
1359 				 10 * HZ);
1360 	if (!ret) {
1361 		dev_err(glink->dev, "intent request timed out\n");
1362 		ret = -ETIMEDOUT;
1363 	} else {
1364 		ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED;
1365 	}
1366 
1367 unlock:
1368 	mutex_unlock(&channel->intent_req_lock);
1369 	return ret;
1370 }
1371 
1372 static int __qcom_glink_send(struct glink_channel *channel,
1373 			     void *data, int len, bool wait)
1374 {
1375 	struct qcom_glink *glink = channel->glink;
1376 	struct glink_core_rx_intent *intent = NULL;
1377 	struct glink_core_rx_intent *tmp;
1378 	int iid = 0;
1379 	struct {
1380 		struct glink_msg msg;
1381 		__le32 chunk_size;
1382 		__le32 left_size;
1383 	} __packed req;
1384 	int ret;
1385 	unsigned long flags;
1386 	int chunk_size = len;
1387 	size_t offset = 0;
1388 
1389 	if (!glink->intentless) {
1390 		while (!intent) {
1391 			spin_lock_irqsave(&channel->intent_lock, flags);
1392 			idr_for_each_entry(&channel->riids, tmp, iid) {
1393 				if (tmp->size >= len && !tmp->in_use) {
1394 					if (!intent)
1395 						intent = tmp;
1396 					else if (intent->size > tmp->size)
1397 						intent = tmp;
1398 					if (intent->size == len)
1399 						break;
1400 				}
1401 			}
1402 			if (intent)
1403 				intent->in_use = true;
1404 			spin_unlock_irqrestore(&channel->intent_lock, flags);
1405 
1406 			/* We found an available intent */
1407 			if (intent)
1408 				break;
1409 
1410 			if (!wait)
1411 				return -EBUSY;
1412 
1413 			ret = qcom_glink_request_intent(glink, channel, len);
1414 			if (ret < 0)
1415 				return ret;
1416 		}
1417 
1418 		iid = intent->id;
1419 	}
1420 
1421 	while (offset < len) {
1422 		chunk_size = len - offset;
1423 		if (chunk_size > SZ_8K && wait)
1424 			chunk_size = SZ_8K;
1425 
1426 		req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1427 		req.msg.param1 = cpu_to_le16(channel->lcid);
1428 		req.msg.param2 = cpu_to_le32(iid);
1429 		req.chunk_size = cpu_to_le32(chunk_size);
1430 		req.left_size = cpu_to_le32(len - offset - chunk_size);
1431 
1432 		ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
1433 		if (ret) {
1434 			/* Mark intent available if we failed */
1435 			if (intent)
1436 				intent->in_use = false;
1437 			return ret;
1438 		}
1439 
1440 		offset += chunk_size;
1441 	}
1442 
1443 	return 0;
1444 }
1445 
1446 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1447 {
1448 	struct glink_channel *channel = to_glink_channel(ept);
1449 
1450 	return __qcom_glink_send(channel, data, len, true);
1451 }
1452 
1453 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1454 {
1455 	struct glink_channel *channel = to_glink_channel(ept);
1456 
1457 	return __qcom_glink_send(channel, data, len, false);
1458 }
1459 
1460 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1461 {
1462 	struct glink_channel *channel = to_glink_channel(ept);
1463 
1464 	return __qcom_glink_send(channel, data, len, true);
1465 }
1466 
1467 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1468 {
1469 	struct glink_channel *channel = to_glink_channel(ept);
1470 
1471 	return __qcom_glink_send(channel, data, len, false);
1472 }
1473 
1474 /*
1475  * Finds the device_node for the glink child interested in this channel.
1476  */
1477 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1478 						    const char *channel)
1479 {
1480 	struct device_node *child;
1481 	const char *name;
1482 	const char *key;
1483 	int ret;
1484 
1485 	for_each_available_child_of_node(node, child) {
1486 		key = "qcom,glink-channels";
1487 		ret = of_property_read_string(child, key, &name);
1488 		if (ret)
1489 			continue;
1490 
1491 		if (strcmp(name, channel) == 0)
1492 			return child;
1493 	}
1494 
1495 	return NULL;
1496 }
1497 
1498 static const struct rpmsg_device_ops glink_device_ops = {
1499 	.create_ept = qcom_glink_create_ept,
1500 	.announce_create = qcom_glink_announce_create,
1501 };
1502 
1503 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1504 	.destroy_ept = qcom_glink_destroy_ept,
1505 	.send = qcom_glink_send,
1506 	.sendto = qcom_glink_sendto,
1507 	.trysend = qcom_glink_trysend,
1508 	.trysendto = qcom_glink_trysendto,
1509 	.set_flow_control = qcom_glink_set_flow_control,
1510 };
1511 
1512 static void qcom_glink_rpdev_release(struct device *dev)
1513 {
1514 	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1515 
1516 	kfree(rpdev->driver_override);
1517 	kfree(rpdev);
1518 }
1519 
1520 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1521 			      char *name)
1522 {
1523 	struct glink_channel *channel;
1524 	struct rpmsg_device *rpdev;
1525 	bool create_device = false;
1526 	struct device_node *node;
1527 	int lcid;
1528 	int ret;
1529 	unsigned long flags;
1530 
1531 	spin_lock_irqsave(&glink->idr_lock, flags);
1532 	idr_for_each_entry(&glink->lcids, channel, lcid) {
1533 		if (!strcmp(channel->name, name))
1534 			break;
1535 	}
1536 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1537 
1538 	if (!channel) {
1539 		channel = qcom_glink_alloc_channel(glink, name);
1540 		if (IS_ERR(channel))
1541 			return PTR_ERR(channel);
1542 
1543 		/* The opening dance was initiated by the remote */
1544 		create_device = true;
1545 	}
1546 
1547 	spin_lock_irqsave(&glink->idr_lock, flags);
1548 	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1549 	if (ret < 0) {
1550 		dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1551 		spin_unlock_irqrestore(&glink->idr_lock, flags);
1552 		goto free_channel;
1553 	}
1554 	channel->rcid = ret;
1555 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1556 
1557 	complete_all(&channel->open_req);
1558 
1559 	if (create_device) {
1560 		rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1561 		if (!rpdev) {
1562 			ret = -ENOMEM;
1563 			goto rcid_remove;
1564 		}
1565 
1566 		rpdev->ept = &channel->ept;
1567 		strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE);
1568 		rpdev->src = RPMSG_ADDR_ANY;
1569 		rpdev->dst = RPMSG_ADDR_ANY;
1570 		rpdev->ops = &glink_device_ops;
1571 
1572 		node = qcom_glink_match_channel(glink->dev->of_node, name);
1573 		rpdev->dev.of_node = node;
1574 		rpdev->dev.parent = glink->dev;
1575 		rpdev->dev.release = qcom_glink_rpdev_release;
1576 
1577 		ret = rpmsg_register_device(rpdev);
1578 		if (ret)
1579 			goto rcid_remove;
1580 
1581 		channel->rpdev = rpdev;
1582 	}
1583 
1584 	return 0;
1585 
1586 rcid_remove:
1587 	spin_lock_irqsave(&glink->idr_lock, flags);
1588 	idr_remove(&glink->rcids, channel->rcid);
1589 	channel->rcid = 0;
1590 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1591 free_channel:
1592 	/* Release the reference, iff we took it */
1593 	if (create_device)
1594 		kref_put(&channel->refcount, qcom_glink_channel_release);
1595 
1596 	return ret;
1597 }
1598 
1599 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1600 {
1601 	struct rpmsg_channel_info chinfo;
1602 	struct glink_channel *channel;
1603 	unsigned long flags;
1604 
1605 	spin_lock_irqsave(&glink->idr_lock, flags);
1606 	channel = idr_find(&glink->rcids, rcid);
1607 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1608 	if (WARN(!channel, "close request on unknown channel\n"))
1609 		return;
1610 
1611 	/* cancel pending rx_done work */
1612 	cancel_work_sync(&channel->intent_work);
1613 
1614 	if (channel->rpdev) {
1615 		strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1616 		chinfo.src = RPMSG_ADDR_ANY;
1617 		chinfo.dst = RPMSG_ADDR_ANY;
1618 
1619 		rpmsg_unregister_device(glink->dev, &chinfo);
1620 	}
1621 	channel->rpdev = NULL;
1622 
1623 	qcom_glink_send_close_ack(glink, channel->rcid);
1624 
1625 	spin_lock_irqsave(&glink->idr_lock, flags);
1626 	idr_remove(&glink->rcids, channel->rcid);
1627 	channel->rcid = 0;
1628 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1629 
1630 	kref_put(&channel->refcount, qcom_glink_channel_release);
1631 }
1632 
1633 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1634 {
1635 	struct rpmsg_channel_info chinfo;
1636 	struct glink_channel *channel;
1637 	unsigned long flags;
1638 
1639 	/* To wakeup any blocking writers */
1640 	wake_up_all(&glink->tx_avail_notify);
1641 
1642 	spin_lock_irqsave(&glink->idr_lock, flags);
1643 	channel = idr_find(&glink->lcids, lcid);
1644 	if (WARN(!channel, "close ack on unknown channel\n")) {
1645 		spin_unlock_irqrestore(&glink->idr_lock, flags);
1646 		return;
1647 	}
1648 
1649 	idr_remove(&glink->lcids, channel->lcid);
1650 	channel->lcid = 0;
1651 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1652 
1653 	/* Decouple the potential rpdev from the channel */
1654 	if (channel->rpdev) {
1655 		strscpy(chinfo.name, channel->name, sizeof(chinfo.name));
1656 		chinfo.src = RPMSG_ADDR_ANY;
1657 		chinfo.dst = RPMSG_ADDR_ANY;
1658 
1659 		rpmsg_unregister_device(glink->dev, &chinfo);
1660 	}
1661 	channel->rpdev = NULL;
1662 
1663 	kref_put(&channel->refcount, qcom_glink_channel_release);
1664 }
1665 
1666 static void qcom_glink_work(struct work_struct *work)
1667 {
1668 	struct qcom_glink *glink = container_of(work, struct qcom_glink,
1669 						rx_work);
1670 	struct glink_defer_cmd *dcmd;
1671 	struct glink_msg *msg;
1672 	unsigned long flags;
1673 	unsigned int param1;
1674 	unsigned int param2;
1675 	unsigned int cmd;
1676 
1677 	for (;;) {
1678 		spin_lock_irqsave(&glink->rx_lock, flags);
1679 		if (list_empty(&glink->rx_queue)) {
1680 			spin_unlock_irqrestore(&glink->rx_lock, flags);
1681 			break;
1682 		}
1683 		dcmd = list_first_entry(&glink->rx_queue,
1684 					struct glink_defer_cmd, node);
1685 		list_del(&dcmd->node);
1686 		spin_unlock_irqrestore(&glink->rx_lock, flags);
1687 
1688 		msg = &dcmd->msg;
1689 		cmd = le16_to_cpu(msg->cmd);
1690 		param1 = le16_to_cpu(msg->param1);
1691 		param2 = le32_to_cpu(msg->param2);
1692 
1693 		switch (cmd) {
1694 		case GLINK_CMD_VERSION:
1695 			qcom_glink_receive_version(glink, param1, param2);
1696 			break;
1697 		case GLINK_CMD_VERSION_ACK:
1698 			qcom_glink_receive_version_ack(glink, param1, param2);
1699 			break;
1700 		case GLINK_CMD_OPEN:
1701 			qcom_glink_rx_open(glink, param1, msg->data);
1702 			break;
1703 		case GLINK_CMD_CLOSE:
1704 			qcom_glink_rx_close(glink, param1);
1705 			break;
1706 		case GLINK_CMD_CLOSE_ACK:
1707 			qcom_glink_rx_close_ack(glink, param1);
1708 			break;
1709 		case GLINK_CMD_RX_INTENT_REQ:
1710 			qcom_glink_handle_intent_req(glink, param1, param2);
1711 			break;
1712 		default:
1713 			WARN(1, "Unknown defer object %d\n", cmd);
1714 			break;
1715 		}
1716 
1717 		kfree(dcmd);
1718 	}
1719 }
1720 
1721 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1722 {
1723 	struct glink_defer_cmd *dcmd;
1724 	struct glink_defer_cmd *tmp;
1725 
1726 	/* cancel any pending deferred rx_work */
1727 	cancel_work_sync(&glink->rx_work);
1728 
1729 	list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1730 		kfree(dcmd);
1731 }
1732 
1733 static ssize_t rpmsg_name_show(struct device *dev,
1734 			       struct device_attribute *attr, char *buf)
1735 {
1736 	int ret = 0;
1737 	const char *name;
1738 
1739 	ret = of_property_read_string(dev->of_node, "label", &name);
1740 	if (ret < 0)
1741 		name = dev->of_node->name;
1742 
1743 	return sysfs_emit(buf, "%s\n", name);
1744 }
1745 static DEVICE_ATTR_RO(rpmsg_name);
1746 
1747 static struct attribute *qcom_glink_attrs[] = {
1748 	&dev_attr_rpmsg_name.attr,
1749 	NULL
1750 };
1751 ATTRIBUTE_GROUPS(qcom_glink);
1752 
1753 static void qcom_glink_device_release(struct device *dev)
1754 {
1755 	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1756 	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1757 
1758 	/* Release qcom_glink_alloc_channel() reference */
1759 	kref_put(&channel->refcount, qcom_glink_channel_release);
1760 	kfree(rpdev->driver_override);
1761 	kfree(rpdev);
1762 }
1763 
1764 static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1765 {
1766 	struct rpmsg_device *rpdev;
1767 	struct glink_channel *channel;
1768 
1769 	rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1770 	if (!rpdev)
1771 		return -ENOMEM;
1772 
1773 	channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1774 	if (IS_ERR(channel)) {
1775 		kfree(rpdev);
1776 		return PTR_ERR(channel);
1777 	}
1778 	channel->rpdev = rpdev;
1779 
1780 	rpdev->ept = &channel->ept;
1781 	rpdev->ops = &glink_device_ops;
1782 	rpdev->dev.parent = glink->dev;
1783 	rpdev->dev.release = qcom_glink_device_release;
1784 
1785 	return rpmsg_ctrldev_register_device(rpdev);
1786 }
1787 
1788 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1789 					   unsigned long features,
1790 					   struct qcom_glink_pipe *rx,
1791 					   struct qcom_glink_pipe *tx,
1792 					   bool intentless)
1793 {
1794 	int ret;
1795 	struct qcom_glink *glink;
1796 
1797 	glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1798 	if (!glink)
1799 		return ERR_PTR(-ENOMEM);
1800 
1801 	glink->dev = dev;
1802 	glink->tx_pipe = tx;
1803 	glink->rx_pipe = rx;
1804 
1805 	glink->features = features;
1806 	glink->intentless = intentless;
1807 
1808 	spin_lock_init(&glink->tx_lock);
1809 	spin_lock_init(&glink->rx_lock);
1810 	INIT_LIST_HEAD(&glink->rx_queue);
1811 	INIT_WORK(&glink->rx_work, qcom_glink_work);
1812 	init_waitqueue_head(&glink->tx_avail_notify);
1813 
1814 	spin_lock_init(&glink->idr_lock);
1815 	idr_init(&glink->lcids);
1816 	idr_init(&glink->rcids);
1817 
1818 	glink->dev->groups = qcom_glink_groups;
1819 
1820 	ret = device_add_groups(dev, qcom_glink_groups);
1821 	if (ret)
1822 		dev_err(dev, "failed to add groups\n");
1823 
1824 	ret = qcom_glink_send_version(glink);
1825 	if (ret)
1826 		return ERR_PTR(ret);
1827 
1828 	ret = qcom_glink_create_chrdev(glink);
1829 	if (ret)
1830 		dev_err(glink->dev, "failed to register chrdev\n");
1831 
1832 	return glink;
1833 }
1834 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1835 
1836 static int qcom_glink_remove_device(struct device *dev, void *data)
1837 {
1838 	device_unregister(dev);
1839 
1840 	return 0;
1841 }
1842 
1843 void qcom_glink_native_remove(struct qcom_glink *glink)
1844 {
1845 	struct glink_channel *channel;
1846 	unsigned long flags;
1847 	int cid;
1848 	int ret;
1849 
1850 	qcom_glink_cancel_rx_work(glink);
1851 
1852 	/* Fail all attempts at sending messages */
1853 	spin_lock_irqsave(&glink->tx_lock, flags);
1854 	glink->abort_tx = true;
1855 	wake_up_all(&glink->tx_avail_notify);
1856 	spin_unlock_irqrestore(&glink->tx_lock, flags);
1857 
1858 	/* Abort any senders waiting for intent requests */
1859 	spin_lock_irqsave(&glink->idr_lock, flags);
1860 	idr_for_each_entry(&glink->lcids, channel, cid)
1861 		qcom_glink_intent_req_abort(channel);
1862 	spin_unlock_irqrestore(&glink->idr_lock, flags);
1863 
1864 	ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1865 	if (ret)
1866 		dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1867 
1868 	/* Release any defunct local channels, waiting for close-ack */
1869 	idr_for_each_entry(&glink->lcids, channel, cid)
1870 		kref_put(&channel->refcount, qcom_glink_channel_release);
1871 
1872 	/* Release any defunct local channels, waiting for close-req */
1873 	idr_for_each_entry(&glink->rcids, channel, cid)
1874 		kref_put(&channel->refcount, qcom_glink_channel_release);
1875 
1876 	idr_destroy(&glink->lcids);
1877 	idr_destroy(&glink->rcids);
1878 }
1879 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1880 
1881 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1882 MODULE_LICENSE("GPL v2");
1883