1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/of_irq.h> 15 #include <linux/platform_device.h> 16 #include <linux/regmap.h> 17 #include <linux/rpmsg.h> 18 #include <linux/sizes.h> 19 #include <linux/slab.h> 20 #include <linux/workqueue.h> 21 #include <linux/mailbox_client.h> 22 23 #include "rpmsg_internal.h" 24 #include "qcom_glink_native.h" 25 26 #define GLINK_NAME_SIZE 32 27 #define GLINK_VERSION_1 1 28 29 #define RPM_GLINK_CID_MIN 1 30 #define RPM_GLINK_CID_MAX 65536 31 32 struct glink_msg { 33 __le16 cmd; 34 __le16 param1; 35 __le32 param2; 36 u8 data[]; 37 } __packed; 38 39 /** 40 * struct glink_defer_cmd - deferred incoming control message 41 * @node: list node 42 * @msg: message header 43 * @data: payload of the message 44 * 45 * Copy of a received control message, to be added to @rx_queue and processed 46 * by @rx_work of @qcom_glink. 47 */ 48 struct glink_defer_cmd { 49 struct list_head node; 50 51 struct glink_msg msg; 52 u8 data[]; 53 }; 54 55 /** 56 * struct glink_core_rx_intent - RX intent 57 * RX intent 58 * 59 * @data: pointer to the data (may be NULL for zero-copy) 60 * @id: remote or local intent ID 61 * @size: size of the original intent (do not modify) 62 * @reuse: To mark if the intent can be reused after first use 63 * @in_use: To mark if intent is already in use for the channel 64 * @offset: next write offset (initially 0) 65 * @node: list node 66 */ 67 struct glink_core_rx_intent { 68 void *data; 69 u32 id; 70 size_t size; 71 bool reuse; 72 bool in_use; 73 u32 offset; 74 75 struct list_head node; 76 }; 77 78 /** 79 * struct qcom_glink - driver context, relates to one remote subsystem 80 * @dev: reference to the associated struct device 81 * @mbox_client: mailbox client 82 * @mbox_chan: mailbox channel 83 * @rx_pipe: pipe object for receive FIFO 84 * @tx_pipe: pipe object for transmit FIFO 85 * @irq: IRQ for signaling incoming events 86 * @rx_work: worker for handling received control messages 87 * @rx_lock: protects the @rx_queue 88 * @rx_queue: queue of received control messages to be processed in @rx_work 89 * @tx_lock: synchronizes operations on the tx fifo 90 * @idr_lock: synchronizes @lcids and @rcids modifications 91 * @lcids: idr of all channels with a known local channel id 92 * @rcids: idr of all channels with a known remote channel id 93 * @features: remote features 94 * @intentless: flag to indicate that there is no intent 95 * @tx_avail_notify: Waitqueue for pending tx tasks 96 * @sent_read_notify: flag to check cmd sent or not 97 */ 98 struct qcom_glink { 99 struct device *dev; 100 101 struct mbox_client mbox_client; 102 struct mbox_chan *mbox_chan; 103 104 struct qcom_glink_pipe *rx_pipe; 105 struct qcom_glink_pipe *tx_pipe; 106 107 int irq; 108 109 struct work_struct rx_work; 110 spinlock_t rx_lock; 111 struct list_head rx_queue; 112 113 spinlock_t tx_lock; 114 115 spinlock_t idr_lock; 116 struct idr lcids; 117 struct idr rcids; 118 unsigned long features; 119 120 bool intentless; 121 wait_queue_head_t tx_avail_notify; 122 bool sent_read_notify; 123 }; 124 125 enum { 126 GLINK_STATE_CLOSED, 127 GLINK_STATE_OPENING, 128 GLINK_STATE_OPEN, 129 GLINK_STATE_CLOSING, 130 }; 131 132 /** 133 * struct glink_channel - internal representation of a channel 134 * @rpdev: rpdev reference, only used for primary endpoints 135 * @ept: rpmsg endpoint this channel is associated with 136 * @glink: qcom_glink context handle 137 * @refcount: refcount for the channel object 138 * @recv_lock: guard for @ept.cb 139 * @name: unique channel name/identifier 140 * @lcid: channel id, in local space 141 * @rcid: channel id, in remote space 142 * @intent_lock: lock for protection of @liids, @riids 143 * @liids: idr of all local intents 144 * @riids: idr of all remote intents 145 * @intent_work: worker responsible for transmitting rx_done packets 146 * @done_intents: list of intents that needs to be announced rx_done 147 * @buf: receive buffer, for gathering fragments 148 * @buf_offset: write offset in @buf 149 * @buf_size: size of current @buf 150 * @open_ack: completed once remote has acked the open-request 151 * @open_req: completed once open-request has been received 152 * @intent_req_lock: Synchronises multiple intent requests 153 * @intent_req_result: Result of intent request 154 * @intent_req_comp: Completion for intent_req signalling 155 */ 156 struct glink_channel { 157 struct rpmsg_endpoint ept; 158 159 struct rpmsg_device *rpdev; 160 struct qcom_glink *glink; 161 162 struct kref refcount; 163 164 spinlock_t recv_lock; 165 166 char *name; 167 unsigned int lcid; 168 unsigned int rcid; 169 170 spinlock_t intent_lock; 171 struct idr liids; 172 struct idr riids; 173 struct work_struct intent_work; 174 struct list_head done_intents; 175 176 struct glink_core_rx_intent *buf; 177 int buf_offset; 178 int buf_size; 179 180 struct completion open_ack; 181 struct completion open_req; 182 183 struct mutex intent_req_lock; 184 bool intent_req_result; 185 struct completion intent_req_comp; 186 }; 187 188 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 189 190 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 191 192 #define RPM_CMD_VERSION 0 193 #define RPM_CMD_VERSION_ACK 1 194 #define RPM_CMD_OPEN 2 195 #define RPM_CMD_CLOSE 3 196 #define RPM_CMD_OPEN_ACK 4 197 #define RPM_CMD_INTENT 5 198 #define RPM_CMD_RX_DONE 6 199 #define RPM_CMD_RX_INTENT_REQ 7 200 #define RPM_CMD_RX_INTENT_REQ_ACK 8 201 #define RPM_CMD_TX_DATA 9 202 #define RPM_CMD_CLOSE_ACK 11 203 #define RPM_CMD_TX_DATA_CONT 12 204 #define RPM_CMD_READ_NOTIF 13 205 #define RPM_CMD_RX_DONE_W_REUSE 14 206 207 #define GLINK_FEATURE_INTENTLESS BIT(1) 208 209 static void qcom_glink_rx_done_work(struct work_struct *work); 210 211 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 212 const char *name) 213 { 214 struct glink_channel *channel; 215 216 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 217 if (!channel) 218 return ERR_PTR(-ENOMEM); 219 220 /* Setup glink internal glink_channel data */ 221 spin_lock_init(&channel->recv_lock); 222 spin_lock_init(&channel->intent_lock); 223 mutex_init(&channel->intent_req_lock); 224 225 channel->glink = glink; 226 channel->name = kstrdup(name, GFP_KERNEL); 227 228 init_completion(&channel->open_req); 229 init_completion(&channel->open_ack); 230 init_completion(&channel->intent_req_comp); 231 232 INIT_LIST_HEAD(&channel->done_intents); 233 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 234 235 idr_init(&channel->liids); 236 idr_init(&channel->riids); 237 kref_init(&channel->refcount); 238 239 return channel; 240 } 241 242 static void qcom_glink_channel_release(struct kref *ref) 243 { 244 struct glink_channel *channel = container_of(ref, struct glink_channel, 245 refcount); 246 struct glink_core_rx_intent *intent; 247 struct glink_core_rx_intent *tmp; 248 unsigned long flags; 249 int iid; 250 251 /* cancel pending rx_done work */ 252 cancel_work_sync(&channel->intent_work); 253 254 spin_lock_irqsave(&channel->intent_lock, flags); 255 /* Free all non-reuse intents pending rx_done work */ 256 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 257 if (!intent->reuse) { 258 kfree(intent->data); 259 kfree(intent); 260 } 261 } 262 263 idr_for_each_entry(&channel->liids, tmp, iid) { 264 kfree(tmp->data); 265 kfree(tmp); 266 } 267 idr_destroy(&channel->liids); 268 269 idr_for_each_entry(&channel->riids, tmp, iid) 270 kfree(tmp); 271 idr_destroy(&channel->riids); 272 spin_unlock_irqrestore(&channel->intent_lock, flags); 273 274 kfree(channel->name); 275 kfree(channel); 276 } 277 278 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 279 { 280 return glink->rx_pipe->avail(glink->rx_pipe); 281 } 282 283 static void qcom_glink_rx_peak(struct qcom_glink *glink, 284 void *data, unsigned int offset, size_t count) 285 { 286 glink->rx_pipe->peak(glink->rx_pipe, data, offset, count); 287 } 288 289 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 290 { 291 glink->rx_pipe->advance(glink->rx_pipe, count); 292 } 293 294 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 295 { 296 return glink->tx_pipe->avail(glink->tx_pipe); 297 } 298 299 static void qcom_glink_tx_write(struct qcom_glink *glink, 300 const void *hdr, size_t hlen, 301 const void *data, size_t dlen) 302 { 303 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 304 } 305 306 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 307 { 308 struct glink_msg msg; 309 310 msg.cmd = cpu_to_le16(RPM_CMD_READ_NOTIF); 311 msg.param1 = 0; 312 msg.param2 = 0; 313 314 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 315 316 mbox_send_message(glink->mbox_chan, NULL); 317 mbox_client_txdone(glink->mbox_chan, 0); 318 } 319 320 static int qcom_glink_tx(struct qcom_glink *glink, 321 const void *hdr, size_t hlen, 322 const void *data, size_t dlen, bool wait) 323 { 324 unsigned int tlen = hlen + dlen; 325 unsigned long flags; 326 int ret = 0; 327 328 /* Reject packets that are too big */ 329 if (tlen >= glink->tx_pipe->length) 330 return -EINVAL; 331 332 spin_lock_irqsave(&glink->tx_lock, flags); 333 334 while (qcom_glink_tx_avail(glink) < tlen) { 335 if (!wait) { 336 ret = -EAGAIN; 337 goto out; 338 } 339 340 if (!glink->sent_read_notify) { 341 glink->sent_read_notify = true; 342 qcom_glink_send_read_notify(glink); 343 } 344 345 /* Wait without holding the tx_lock */ 346 spin_unlock_irqrestore(&glink->tx_lock, flags); 347 348 wait_event_timeout(glink->tx_avail_notify, 349 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 350 351 spin_lock_irqsave(&glink->tx_lock, flags); 352 353 if (qcom_glink_tx_avail(glink) >= tlen) 354 glink->sent_read_notify = false; 355 } 356 357 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 358 359 mbox_send_message(glink->mbox_chan, NULL); 360 mbox_client_txdone(glink->mbox_chan, 0); 361 362 out: 363 spin_unlock_irqrestore(&glink->tx_lock, flags); 364 365 return ret; 366 } 367 368 static int qcom_glink_send_version(struct qcom_glink *glink) 369 { 370 struct glink_msg msg; 371 372 msg.cmd = cpu_to_le16(RPM_CMD_VERSION); 373 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 374 msg.param2 = cpu_to_le32(glink->features); 375 376 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 377 } 378 379 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 380 { 381 struct glink_msg msg; 382 383 msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK); 384 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 385 msg.param2 = cpu_to_le32(glink->features); 386 387 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 388 } 389 390 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 391 struct glink_channel *channel) 392 { 393 struct glink_msg msg; 394 395 msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK); 396 msg.param1 = cpu_to_le16(channel->rcid); 397 msg.param2 = cpu_to_le32(0); 398 399 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 400 } 401 402 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 403 unsigned int cid, bool granted) 404 { 405 struct glink_channel *channel; 406 unsigned long flags; 407 408 spin_lock_irqsave(&glink->idr_lock, flags); 409 channel = idr_find(&glink->rcids, cid); 410 spin_unlock_irqrestore(&glink->idr_lock, flags); 411 if (!channel) { 412 dev_err(glink->dev, "unable to find channel\n"); 413 return; 414 } 415 416 channel->intent_req_result = granted; 417 complete(&channel->intent_req_comp); 418 } 419 420 /** 421 * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote 422 * @glink: Ptr to the glink edge 423 * @channel: Ptr to the channel that the open req is sent 424 * 425 * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote. 426 * Will return with refcount held, regardless of outcome. 427 * 428 * Return: 0 on success, negative errno otherwise. 429 */ 430 static int qcom_glink_send_open_req(struct qcom_glink *glink, 431 struct glink_channel *channel) 432 { 433 struct { 434 struct glink_msg msg; 435 u8 name[GLINK_NAME_SIZE]; 436 } __packed req; 437 int name_len = strlen(channel->name) + 1; 438 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 439 int ret; 440 unsigned long flags; 441 442 kref_get(&channel->refcount); 443 444 spin_lock_irqsave(&glink->idr_lock, flags); 445 ret = idr_alloc_cyclic(&glink->lcids, channel, 446 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 447 GFP_ATOMIC); 448 spin_unlock_irqrestore(&glink->idr_lock, flags); 449 if (ret < 0) 450 return ret; 451 452 channel->lcid = ret; 453 454 req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN); 455 req.msg.param1 = cpu_to_le16(channel->lcid); 456 req.msg.param2 = cpu_to_le32(name_len); 457 strcpy(req.name, channel->name); 458 459 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 460 if (ret) 461 goto remove_idr; 462 463 return 0; 464 465 remove_idr: 466 spin_lock_irqsave(&glink->idr_lock, flags); 467 idr_remove(&glink->lcids, channel->lcid); 468 channel->lcid = 0; 469 spin_unlock_irqrestore(&glink->idr_lock, flags); 470 471 return ret; 472 } 473 474 static void qcom_glink_send_close_req(struct qcom_glink *glink, 475 struct glink_channel *channel) 476 { 477 struct glink_msg req; 478 479 req.cmd = cpu_to_le16(RPM_CMD_CLOSE); 480 req.param1 = cpu_to_le16(channel->lcid); 481 req.param2 = 0; 482 483 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 484 } 485 486 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 487 unsigned int rcid) 488 { 489 struct glink_msg req; 490 491 req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK); 492 req.param1 = cpu_to_le16(rcid); 493 req.param2 = 0; 494 495 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 496 } 497 498 static void qcom_glink_rx_done_work(struct work_struct *work) 499 { 500 struct glink_channel *channel = container_of(work, struct glink_channel, 501 intent_work); 502 struct qcom_glink *glink = channel->glink; 503 struct glink_core_rx_intent *intent, *tmp; 504 struct { 505 u16 id; 506 u16 lcid; 507 u32 liid; 508 } __packed cmd; 509 510 unsigned int cid = channel->lcid; 511 unsigned int iid; 512 bool reuse; 513 unsigned long flags; 514 515 spin_lock_irqsave(&channel->intent_lock, flags); 516 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 517 list_del(&intent->node); 518 spin_unlock_irqrestore(&channel->intent_lock, flags); 519 iid = intent->id; 520 reuse = intent->reuse; 521 522 cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE; 523 cmd.lcid = cid; 524 cmd.liid = iid; 525 526 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 527 if (!reuse) { 528 kfree(intent->data); 529 kfree(intent); 530 } 531 spin_lock_irqsave(&channel->intent_lock, flags); 532 } 533 spin_unlock_irqrestore(&channel->intent_lock, flags); 534 } 535 536 static void qcom_glink_rx_done(struct qcom_glink *glink, 537 struct glink_channel *channel, 538 struct glink_core_rx_intent *intent) 539 { 540 /* We don't send RX_DONE to intentless systems */ 541 if (glink->intentless) { 542 kfree(intent->data); 543 kfree(intent); 544 return; 545 } 546 547 /* Take it off the tree of receive intents */ 548 if (!intent->reuse) { 549 spin_lock(&channel->intent_lock); 550 idr_remove(&channel->liids, intent->id); 551 spin_unlock(&channel->intent_lock); 552 } 553 554 /* Schedule the sending of a rx_done indication */ 555 spin_lock(&channel->intent_lock); 556 list_add_tail(&intent->node, &channel->done_intents); 557 spin_unlock(&channel->intent_lock); 558 559 schedule_work(&channel->intent_work); 560 } 561 562 /** 563 * qcom_glink_receive_version() - receive version/features from remote system 564 * 565 * @glink: pointer to transport interface 566 * @version: remote version 567 * @features: remote features 568 * 569 * This function is called in response to a remote-initiated version/feature 570 * negotiation sequence. 571 */ 572 static void qcom_glink_receive_version(struct qcom_glink *glink, 573 u32 version, 574 u32 features) 575 { 576 switch (version) { 577 case 0: 578 break; 579 case GLINK_VERSION_1: 580 glink->features &= features; 581 fallthrough; 582 default: 583 qcom_glink_send_version_ack(glink); 584 break; 585 } 586 } 587 588 /** 589 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 590 * 591 * @glink: pointer to transport interface 592 * @version: remote version response 593 * @features: remote features response 594 * 595 * This function is called in response to a local-initiated version/feature 596 * negotiation sequence and is the counter-offer from the remote side based 597 * upon the initial version and feature set requested. 598 */ 599 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 600 u32 version, 601 u32 features) 602 { 603 switch (version) { 604 case 0: 605 /* Version negotiation failed */ 606 break; 607 case GLINK_VERSION_1: 608 if (features == glink->features) 609 break; 610 611 glink->features &= features; 612 fallthrough; 613 default: 614 qcom_glink_send_version(glink); 615 break; 616 } 617 } 618 619 /** 620 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 621 * wire format and transmit 622 * @glink: The transport to transmit on. 623 * @channel: The glink channel 624 * @granted: The request response to encode. 625 * 626 * Return: 0 on success or standard Linux error code. 627 */ 628 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 629 struct glink_channel *channel, 630 bool granted) 631 { 632 struct glink_msg msg; 633 634 msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK); 635 msg.param1 = cpu_to_le16(channel->lcid); 636 msg.param2 = cpu_to_le32(granted); 637 638 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 639 640 return 0; 641 } 642 643 /** 644 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 645 * transmit 646 * @glink: The transport to transmit on. 647 * @channel: The local channel 648 * @intent: The intent to pass on to remote. 649 * 650 * Return: 0 on success or standard Linux error code. 651 */ 652 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 653 struct glink_channel *channel, 654 struct glink_core_rx_intent *intent) 655 { 656 struct command { 657 __le16 id; 658 __le16 lcid; 659 __le32 count; 660 __le32 size; 661 __le32 liid; 662 } __packed; 663 struct command cmd; 664 665 cmd.id = cpu_to_le16(RPM_CMD_INTENT); 666 cmd.lcid = cpu_to_le16(channel->lcid); 667 cmd.count = cpu_to_le32(1); 668 cmd.size = cpu_to_le32(intent->size); 669 cmd.liid = cpu_to_le32(intent->id); 670 671 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 672 673 return 0; 674 } 675 676 static struct glink_core_rx_intent * 677 qcom_glink_alloc_intent(struct qcom_glink *glink, 678 struct glink_channel *channel, 679 size_t size, 680 bool reuseable) 681 { 682 struct glink_core_rx_intent *intent; 683 int ret; 684 unsigned long flags; 685 686 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 687 if (!intent) 688 return NULL; 689 690 intent->data = kzalloc(size, GFP_KERNEL); 691 if (!intent->data) 692 goto free_intent; 693 694 spin_lock_irqsave(&channel->intent_lock, flags); 695 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 696 if (ret < 0) { 697 spin_unlock_irqrestore(&channel->intent_lock, flags); 698 goto free_data; 699 } 700 spin_unlock_irqrestore(&channel->intent_lock, flags); 701 702 intent->id = ret; 703 intent->size = size; 704 intent->reuse = reuseable; 705 706 return intent; 707 708 free_data: 709 kfree(intent->data); 710 free_intent: 711 kfree(intent); 712 return NULL; 713 } 714 715 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 716 u32 cid, uint32_t iid, 717 bool reuse) 718 { 719 struct glink_core_rx_intent *intent; 720 struct glink_channel *channel; 721 unsigned long flags; 722 723 spin_lock_irqsave(&glink->idr_lock, flags); 724 channel = idr_find(&glink->rcids, cid); 725 spin_unlock_irqrestore(&glink->idr_lock, flags); 726 if (!channel) { 727 dev_err(glink->dev, "invalid channel id received\n"); 728 return; 729 } 730 731 spin_lock_irqsave(&channel->intent_lock, flags); 732 intent = idr_find(&channel->riids, iid); 733 734 if (!intent) { 735 spin_unlock_irqrestore(&channel->intent_lock, flags); 736 dev_err(glink->dev, "invalid intent id received\n"); 737 return; 738 } 739 740 intent->in_use = false; 741 742 if (!reuse) { 743 idr_remove(&channel->riids, intent->id); 744 kfree(intent); 745 } 746 spin_unlock_irqrestore(&channel->intent_lock, flags); 747 } 748 749 /** 750 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 751 * from remote side 752 * @glink: Pointer to the transport interface 753 * @cid: Remote channel ID 754 * @size: size of the intent 755 * 756 * The function searches for the local channel to which the request for 757 * rx_intent has arrived and allocates and notifies the remote back 758 */ 759 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 760 u32 cid, size_t size) 761 { 762 struct glink_core_rx_intent *intent; 763 struct glink_channel *channel; 764 unsigned long flags; 765 766 spin_lock_irqsave(&glink->idr_lock, flags); 767 channel = idr_find(&glink->rcids, cid); 768 spin_unlock_irqrestore(&glink->idr_lock, flags); 769 770 if (!channel) { 771 pr_err("%s channel not found for cid %d\n", __func__, cid); 772 return; 773 } 774 775 intent = qcom_glink_alloc_intent(glink, channel, size, false); 776 if (intent) 777 qcom_glink_advertise_intent(glink, channel, intent); 778 779 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 780 } 781 782 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 783 { 784 struct glink_defer_cmd *dcmd; 785 786 extra = ALIGN(extra, 8); 787 788 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 789 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 790 return -ENXIO; 791 } 792 793 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 794 if (!dcmd) 795 return -ENOMEM; 796 797 INIT_LIST_HEAD(&dcmd->node); 798 799 qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 800 801 spin_lock(&glink->rx_lock); 802 list_add_tail(&dcmd->node, &glink->rx_queue); 803 spin_unlock(&glink->rx_lock); 804 805 schedule_work(&glink->rx_work); 806 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 807 808 return 0; 809 } 810 811 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 812 { 813 struct glink_core_rx_intent *intent; 814 struct glink_channel *channel; 815 struct { 816 struct glink_msg msg; 817 __le32 chunk_size; 818 __le32 left_size; 819 } __packed hdr; 820 unsigned int chunk_size; 821 unsigned int left_size; 822 unsigned int rcid; 823 unsigned int liid; 824 int ret = 0; 825 unsigned long flags; 826 827 if (avail < sizeof(hdr)) { 828 dev_dbg(glink->dev, "Not enough data in fifo\n"); 829 return -EAGAIN; 830 } 831 832 qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr)); 833 chunk_size = le32_to_cpu(hdr.chunk_size); 834 left_size = le32_to_cpu(hdr.left_size); 835 836 if (avail < sizeof(hdr) + chunk_size) { 837 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 838 return -EAGAIN; 839 } 840 841 rcid = le16_to_cpu(hdr.msg.param1); 842 spin_lock_irqsave(&glink->idr_lock, flags); 843 channel = idr_find(&glink->rcids, rcid); 844 spin_unlock_irqrestore(&glink->idr_lock, flags); 845 if (!channel) { 846 dev_dbg(glink->dev, "Data on non-existing channel\n"); 847 848 /* Drop the message */ 849 goto advance_rx; 850 } 851 852 if (glink->intentless) { 853 /* Might have an ongoing, fragmented, message to append */ 854 if (!channel->buf) { 855 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 856 if (!intent) 857 return -ENOMEM; 858 859 intent->data = kmalloc(chunk_size + left_size, 860 GFP_ATOMIC); 861 if (!intent->data) { 862 kfree(intent); 863 return -ENOMEM; 864 } 865 866 intent->id = 0xbabababa; 867 intent->size = chunk_size + left_size; 868 intent->offset = 0; 869 870 channel->buf = intent; 871 } else { 872 intent = channel->buf; 873 } 874 } else { 875 liid = le32_to_cpu(hdr.msg.param2); 876 877 spin_lock_irqsave(&channel->intent_lock, flags); 878 intent = idr_find(&channel->liids, liid); 879 spin_unlock_irqrestore(&channel->intent_lock, flags); 880 881 if (!intent) { 882 dev_err(glink->dev, 883 "no intent found for channel %s intent %d", 884 channel->name, liid); 885 ret = -ENOENT; 886 goto advance_rx; 887 } 888 } 889 890 if (intent->size - intent->offset < chunk_size) { 891 dev_err(glink->dev, "Insufficient space in intent\n"); 892 893 /* The packet header lied, drop payload */ 894 goto advance_rx; 895 } 896 897 qcom_glink_rx_peak(glink, intent->data + intent->offset, 898 sizeof(hdr), chunk_size); 899 intent->offset += chunk_size; 900 901 /* Handle message when no fragments remain to be received */ 902 if (!left_size) { 903 spin_lock(&channel->recv_lock); 904 if (channel->ept.cb) { 905 channel->ept.cb(channel->ept.rpdev, 906 intent->data, 907 intent->offset, 908 channel->ept.priv, 909 RPMSG_ADDR_ANY); 910 } 911 spin_unlock(&channel->recv_lock); 912 913 intent->offset = 0; 914 channel->buf = NULL; 915 916 qcom_glink_rx_done(glink, channel, intent); 917 } 918 919 advance_rx: 920 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 921 922 return ret; 923 } 924 925 static void qcom_glink_handle_intent(struct qcom_glink *glink, 926 unsigned int cid, 927 unsigned int count, 928 size_t avail) 929 { 930 struct glink_core_rx_intent *intent; 931 struct glink_channel *channel; 932 struct intent_pair { 933 __le32 size; 934 __le32 iid; 935 }; 936 937 struct { 938 struct glink_msg msg; 939 struct intent_pair intents[]; 940 } __packed * msg; 941 942 const size_t msglen = struct_size(msg, intents, count); 943 int ret; 944 int i; 945 unsigned long flags; 946 947 if (avail < msglen) { 948 dev_dbg(glink->dev, "Not enough data in fifo\n"); 949 return; 950 } 951 952 spin_lock_irqsave(&glink->idr_lock, flags); 953 channel = idr_find(&glink->rcids, cid); 954 spin_unlock_irqrestore(&glink->idr_lock, flags); 955 if (!channel) { 956 dev_err(glink->dev, "intents for non-existing channel\n"); 957 return; 958 } 959 960 msg = kmalloc(msglen, GFP_ATOMIC); 961 if (!msg) 962 return; 963 964 qcom_glink_rx_peak(glink, msg, 0, msglen); 965 966 for (i = 0; i < count; ++i) { 967 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 968 if (!intent) 969 break; 970 971 intent->id = le32_to_cpu(msg->intents[i].iid); 972 intent->size = le32_to_cpu(msg->intents[i].size); 973 974 spin_lock_irqsave(&channel->intent_lock, flags); 975 ret = idr_alloc(&channel->riids, intent, 976 intent->id, intent->id + 1, GFP_ATOMIC); 977 spin_unlock_irqrestore(&channel->intent_lock, flags); 978 979 if (ret < 0) 980 dev_err(glink->dev, "failed to store remote intent\n"); 981 } 982 983 kfree(msg); 984 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 985 } 986 987 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 988 { 989 struct glink_channel *channel; 990 991 spin_lock(&glink->idr_lock); 992 channel = idr_find(&glink->lcids, lcid); 993 spin_unlock(&glink->idr_lock); 994 if (!channel) { 995 dev_err(glink->dev, "Invalid open ack packet\n"); 996 return -EINVAL; 997 } 998 999 complete_all(&channel->open_ack); 1000 1001 return 0; 1002 } 1003 1004 static irqreturn_t qcom_glink_native_intr(int irq, void *data) 1005 { 1006 struct qcom_glink *glink = data; 1007 struct glink_msg msg; 1008 unsigned int param1; 1009 unsigned int param2; 1010 unsigned int avail; 1011 unsigned int cmd; 1012 int ret = 0; 1013 1014 /* To wakeup any blocking writers */ 1015 wake_up_all(&glink->tx_avail_notify); 1016 1017 for (;;) { 1018 avail = qcom_glink_rx_avail(glink); 1019 if (avail < sizeof(msg)) 1020 break; 1021 1022 qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg)); 1023 1024 cmd = le16_to_cpu(msg.cmd); 1025 param1 = le16_to_cpu(msg.param1); 1026 param2 = le32_to_cpu(msg.param2); 1027 1028 switch (cmd) { 1029 case RPM_CMD_VERSION: 1030 case RPM_CMD_VERSION_ACK: 1031 case RPM_CMD_CLOSE: 1032 case RPM_CMD_CLOSE_ACK: 1033 case RPM_CMD_RX_INTENT_REQ: 1034 ret = qcom_glink_rx_defer(glink, 0); 1035 break; 1036 case RPM_CMD_OPEN_ACK: 1037 ret = qcom_glink_rx_open_ack(glink, param1); 1038 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1039 break; 1040 case RPM_CMD_OPEN: 1041 ret = qcom_glink_rx_defer(glink, param2); 1042 break; 1043 case RPM_CMD_TX_DATA: 1044 case RPM_CMD_TX_DATA_CONT: 1045 ret = qcom_glink_rx_data(glink, avail); 1046 break; 1047 case RPM_CMD_READ_NOTIF: 1048 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1049 1050 mbox_send_message(glink->mbox_chan, NULL); 1051 mbox_client_txdone(glink->mbox_chan, 0); 1052 break; 1053 case RPM_CMD_INTENT: 1054 qcom_glink_handle_intent(glink, param1, param2, avail); 1055 break; 1056 case RPM_CMD_RX_DONE: 1057 qcom_glink_handle_rx_done(glink, param1, param2, false); 1058 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1059 break; 1060 case RPM_CMD_RX_DONE_W_REUSE: 1061 qcom_glink_handle_rx_done(glink, param1, param2, true); 1062 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1063 break; 1064 case RPM_CMD_RX_INTENT_REQ_ACK: 1065 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1066 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1067 break; 1068 default: 1069 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1070 ret = -EINVAL; 1071 break; 1072 } 1073 1074 if (ret) 1075 break; 1076 } 1077 1078 return IRQ_HANDLED; 1079 } 1080 1081 /* Locally initiated rpmsg_create_ept */ 1082 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1083 const char *name) 1084 { 1085 struct glink_channel *channel; 1086 int ret; 1087 unsigned long flags; 1088 1089 channel = qcom_glink_alloc_channel(glink, name); 1090 if (IS_ERR(channel)) 1091 return ERR_CAST(channel); 1092 1093 ret = qcom_glink_send_open_req(glink, channel); 1094 if (ret) 1095 goto release_channel; 1096 1097 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1098 if (!ret) 1099 goto err_timeout; 1100 1101 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1102 if (!ret) 1103 goto err_timeout; 1104 1105 qcom_glink_send_open_ack(glink, channel); 1106 1107 return channel; 1108 1109 err_timeout: 1110 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1111 spin_lock_irqsave(&glink->idr_lock, flags); 1112 idr_remove(&glink->lcids, channel->lcid); 1113 spin_unlock_irqrestore(&glink->idr_lock, flags); 1114 1115 release_channel: 1116 /* Release qcom_glink_send_open_req() reference */ 1117 kref_put(&channel->refcount, qcom_glink_channel_release); 1118 /* Release qcom_glink_alloc_channel() reference */ 1119 kref_put(&channel->refcount, qcom_glink_channel_release); 1120 1121 return ERR_PTR(-ETIMEDOUT); 1122 } 1123 1124 /* Remote initiated rpmsg_create_ept */ 1125 static int qcom_glink_create_remote(struct qcom_glink *glink, 1126 struct glink_channel *channel) 1127 { 1128 int ret; 1129 1130 qcom_glink_send_open_ack(glink, channel); 1131 1132 ret = qcom_glink_send_open_req(glink, channel); 1133 if (ret) 1134 goto close_link; 1135 1136 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1137 if (!ret) { 1138 ret = -ETIMEDOUT; 1139 goto close_link; 1140 } 1141 1142 return 0; 1143 1144 close_link: 1145 /* 1146 * Send a close request to "undo" our open-ack. The close-ack will 1147 * release qcom_glink_send_open_req() reference and the last reference 1148 * will be relesed after receiving remote_close or transport unregister 1149 * by calling qcom_glink_native_remove(). 1150 */ 1151 qcom_glink_send_close_req(glink, channel); 1152 1153 return ret; 1154 } 1155 1156 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1157 rpmsg_rx_cb_t cb, 1158 void *priv, 1159 struct rpmsg_channel_info 1160 chinfo) 1161 { 1162 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1163 struct glink_channel *channel; 1164 struct qcom_glink *glink = parent->glink; 1165 struct rpmsg_endpoint *ept; 1166 const char *name = chinfo.name; 1167 int cid; 1168 int ret; 1169 unsigned long flags; 1170 1171 spin_lock_irqsave(&glink->idr_lock, flags); 1172 idr_for_each_entry(&glink->rcids, channel, cid) { 1173 if (!strcmp(channel->name, name)) 1174 break; 1175 } 1176 spin_unlock_irqrestore(&glink->idr_lock, flags); 1177 1178 if (!channel) { 1179 channel = qcom_glink_create_local(glink, name); 1180 if (IS_ERR(channel)) 1181 return NULL; 1182 } else { 1183 ret = qcom_glink_create_remote(glink, channel); 1184 if (ret) 1185 return NULL; 1186 } 1187 1188 ept = &channel->ept; 1189 ept->rpdev = rpdev; 1190 ept->cb = cb; 1191 ept->priv = priv; 1192 ept->ops = &glink_endpoint_ops; 1193 1194 return ept; 1195 } 1196 1197 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1198 { 1199 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1200 struct device_node *np = rpdev->dev.of_node; 1201 struct qcom_glink *glink = channel->glink; 1202 struct glink_core_rx_intent *intent; 1203 const struct property *prop = NULL; 1204 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1205 int num_intents; 1206 int num_groups = 1; 1207 __be32 *val = defaults; 1208 int size; 1209 1210 if (glink->intentless || !completion_done(&channel->open_ack)) 1211 return 0; 1212 1213 prop = of_find_property(np, "qcom,intents", NULL); 1214 if (prop) { 1215 val = prop->value; 1216 num_groups = prop->length / sizeof(u32) / 2; 1217 } 1218 1219 /* Channel is now open, advertise base set of intents */ 1220 while (num_groups--) { 1221 size = be32_to_cpup(val++); 1222 num_intents = be32_to_cpup(val++); 1223 while (num_intents--) { 1224 intent = qcom_glink_alloc_intent(glink, channel, size, 1225 true); 1226 if (!intent) 1227 break; 1228 1229 qcom_glink_advertise_intent(glink, channel, intent); 1230 } 1231 } 1232 return 0; 1233 } 1234 1235 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1236 { 1237 struct glink_channel *channel = to_glink_channel(ept); 1238 struct qcom_glink *glink = channel->glink; 1239 unsigned long flags; 1240 1241 spin_lock_irqsave(&channel->recv_lock, flags); 1242 channel->ept.cb = NULL; 1243 spin_unlock_irqrestore(&channel->recv_lock, flags); 1244 1245 /* Decouple the potential rpdev from the channel */ 1246 channel->rpdev = NULL; 1247 1248 qcom_glink_send_close_req(glink, channel); 1249 } 1250 1251 static int qcom_glink_request_intent(struct qcom_glink *glink, 1252 struct glink_channel *channel, 1253 size_t size) 1254 { 1255 struct { 1256 u16 id; 1257 u16 cid; 1258 u32 size; 1259 } __packed cmd; 1260 1261 int ret; 1262 1263 mutex_lock(&channel->intent_req_lock); 1264 1265 reinit_completion(&channel->intent_req_comp); 1266 1267 cmd.id = RPM_CMD_RX_INTENT_REQ; 1268 cmd.cid = channel->lcid; 1269 cmd.size = size; 1270 1271 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1272 if (ret) 1273 goto unlock; 1274 1275 ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ); 1276 if (!ret) { 1277 dev_err(glink->dev, "intent request timed out\n"); 1278 ret = -ETIMEDOUT; 1279 } else { 1280 ret = channel->intent_req_result ? 0 : -ECANCELED; 1281 } 1282 1283 unlock: 1284 mutex_unlock(&channel->intent_req_lock); 1285 return ret; 1286 } 1287 1288 static int __qcom_glink_send(struct glink_channel *channel, 1289 void *data, int len, bool wait) 1290 { 1291 struct qcom_glink *glink = channel->glink; 1292 struct glink_core_rx_intent *intent = NULL; 1293 struct glink_core_rx_intent *tmp; 1294 int iid = 0; 1295 struct { 1296 struct glink_msg msg; 1297 __le32 chunk_size; 1298 __le32 left_size; 1299 } __packed req; 1300 int ret; 1301 unsigned long flags; 1302 int chunk_size = len; 1303 int left_size = 0; 1304 1305 if (!glink->intentless) { 1306 while (!intent) { 1307 spin_lock_irqsave(&channel->intent_lock, flags); 1308 idr_for_each_entry(&channel->riids, tmp, iid) { 1309 if (tmp->size >= len && !tmp->in_use) { 1310 if (!intent) 1311 intent = tmp; 1312 else if (intent->size > tmp->size) 1313 intent = tmp; 1314 if (intent->size == len) 1315 break; 1316 } 1317 } 1318 if (intent) 1319 intent->in_use = true; 1320 spin_unlock_irqrestore(&channel->intent_lock, flags); 1321 1322 /* We found an available intent */ 1323 if (intent) 1324 break; 1325 1326 if (!wait) 1327 return -EBUSY; 1328 1329 ret = qcom_glink_request_intent(glink, channel, len); 1330 if (ret < 0) 1331 return ret; 1332 } 1333 1334 iid = intent->id; 1335 } 1336 1337 if (wait && chunk_size > SZ_8K) { 1338 chunk_size = SZ_8K; 1339 left_size = len - chunk_size; 1340 } 1341 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA); 1342 req.msg.param1 = cpu_to_le16(channel->lcid); 1343 req.msg.param2 = cpu_to_le32(iid); 1344 req.chunk_size = cpu_to_le32(chunk_size); 1345 req.left_size = cpu_to_le32(left_size); 1346 1347 ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait); 1348 1349 /* Mark intent available if we failed */ 1350 if (ret && intent) { 1351 intent->in_use = false; 1352 return ret; 1353 } 1354 1355 while (left_size > 0) { 1356 data = (void *)((char *)data + chunk_size); 1357 chunk_size = left_size; 1358 if (chunk_size > SZ_8K) 1359 chunk_size = SZ_8K; 1360 left_size -= chunk_size; 1361 1362 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA_CONT); 1363 req.msg.param1 = cpu_to_le16(channel->lcid); 1364 req.msg.param2 = cpu_to_le32(iid); 1365 req.chunk_size = cpu_to_le32(chunk_size); 1366 req.left_size = cpu_to_le32(left_size); 1367 1368 ret = qcom_glink_tx(glink, &req, sizeof(req), data, 1369 chunk_size, wait); 1370 1371 /* Mark intent available if we failed */ 1372 if (ret && intent) { 1373 intent->in_use = false; 1374 break; 1375 } 1376 } 1377 return ret; 1378 } 1379 1380 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1381 { 1382 struct glink_channel *channel = to_glink_channel(ept); 1383 1384 return __qcom_glink_send(channel, data, len, true); 1385 } 1386 1387 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1388 { 1389 struct glink_channel *channel = to_glink_channel(ept); 1390 1391 return __qcom_glink_send(channel, data, len, false); 1392 } 1393 1394 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1395 { 1396 struct glink_channel *channel = to_glink_channel(ept); 1397 1398 return __qcom_glink_send(channel, data, len, true); 1399 } 1400 1401 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1402 { 1403 struct glink_channel *channel = to_glink_channel(ept); 1404 1405 return __qcom_glink_send(channel, data, len, false); 1406 } 1407 1408 /* 1409 * Finds the device_node for the glink child interested in this channel. 1410 */ 1411 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1412 const char *channel) 1413 { 1414 struct device_node *child; 1415 const char *name; 1416 const char *key; 1417 int ret; 1418 1419 for_each_available_child_of_node(node, child) { 1420 key = "qcom,glink-channels"; 1421 ret = of_property_read_string(child, key, &name); 1422 if (ret) 1423 continue; 1424 1425 if (strcmp(name, channel) == 0) 1426 return child; 1427 } 1428 1429 return NULL; 1430 } 1431 1432 static const struct rpmsg_device_ops glink_device_ops = { 1433 .create_ept = qcom_glink_create_ept, 1434 .announce_create = qcom_glink_announce_create, 1435 }; 1436 1437 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1438 .destroy_ept = qcom_glink_destroy_ept, 1439 .send = qcom_glink_send, 1440 .sendto = qcom_glink_sendto, 1441 .trysend = qcom_glink_trysend, 1442 .trysendto = qcom_glink_trysendto, 1443 }; 1444 1445 static void qcom_glink_rpdev_release(struct device *dev) 1446 { 1447 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1448 1449 kfree(rpdev); 1450 } 1451 1452 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1453 char *name) 1454 { 1455 struct glink_channel *channel; 1456 struct rpmsg_device *rpdev; 1457 bool create_device = false; 1458 struct device_node *node; 1459 int lcid; 1460 int ret; 1461 unsigned long flags; 1462 1463 spin_lock_irqsave(&glink->idr_lock, flags); 1464 idr_for_each_entry(&glink->lcids, channel, lcid) { 1465 if (!strcmp(channel->name, name)) 1466 break; 1467 } 1468 spin_unlock_irqrestore(&glink->idr_lock, flags); 1469 1470 if (!channel) { 1471 channel = qcom_glink_alloc_channel(glink, name); 1472 if (IS_ERR(channel)) 1473 return PTR_ERR(channel); 1474 1475 /* The opening dance was initiated by the remote */ 1476 create_device = true; 1477 } 1478 1479 spin_lock_irqsave(&glink->idr_lock, flags); 1480 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1481 if (ret < 0) { 1482 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1483 spin_unlock_irqrestore(&glink->idr_lock, flags); 1484 goto free_channel; 1485 } 1486 channel->rcid = ret; 1487 spin_unlock_irqrestore(&glink->idr_lock, flags); 1488 1489 complete_all(&channel->open_req); 1490 1491 if (create_device) { 1492 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1493 if (!rpdev) { 1494 ret = -ENOMEM; 1495 goto rcid_remove; 1496 } 1497 1498 rpdev->ept = &channel->ept; 1499 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1500 rpdev->src = RPMSG_ADDR_ANY; 1501 rpdev->dst = RPMSG_ADDR_ANY; 1502 rpdev->ops = &glink_device_ops; 1503 1504 node = qcom_glink_match_channel(glink->dev->of_node, name); 1505 rpdev->dev.of_node = node; 1506 rpdev->dev.parent = glink->dev; 1507 rpdev->dev.release = qcom_glink_rpdev_release; 1508 1509 ret = rpmsg_register_device(rpdev); 1510 if (ret) 1511 goto rcid_remove; 1512 1513 channel->rpdev = rpdev; 1514 } 1515 1516 return 0; 1517 1518 rcid_remove: 1519 spin_lock_irqsave(&glink->idr_lock, flags); 1520 idr_remove(&glink->rcids, channel->rcid); 1521 channel->rcid = 0; 1522 spin_unlock_irqrestore(&glink->idr_lock, flags); 1523 free_channel: 1524 /* Release the reference, iff we took it */ 1525 if (create_device) 1526 kref_put(&channel->refcount, qcom_glink_channel_release); 1527 1528 return ret; 1529 } 1530 1531 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1532 { 1533 struct rpmsg_channel_info chinfo; 1534 struct glink_channel *channel; 1535 unsigned long flags; 1536 1537 spin_lock_irqsave(&glink->idr_lock, flags); 1538 channel = idr_find(&glink->rcids, rcid); 1539 spin_unlock_irqrestore(&glink->idr_lock, flags); 1540 if (WARN(!channel, "close request on unknown channel\n")) 1541 return; 1542 1543 /* cancel pending rx_done work */ 1544 cancel_work_sync(&channel->intent_work); 1545 1546 if (channel->rpdev) { 1547 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1548 chinfo.src = RPMSG_ADDR_ANY; 1549 chinfo.dst = RPMSG_ADDR_ANY; 1550 1551 rpmsg_unregister_device(glink->dev, &chinfo); 1552 } 1553 channel->rpdev = NULL; 1554 1555 qcom_glink_send_close_ack(glink, channel->rcid); 1556 1557 spin_lock_irqsave(&glink->idr_lock, flags); 1558 idr_remove(&glink->rcids, channel->rcid); 1559 channel->rcid = 0; 1560 spin_unlock_irqrestore(&glink->idr_lock, flags); 1561 1562 kref_put(&channel->refcount, qcom_glink_channel_release); 1563 } 1564 1565 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1566 { 1567 struct rpmsg_channel_info chinfo; 1568 struct glink_channel *channel; 1569 unsigned long flags; 1570 1571 /* To wakeup any blocking writers */ 1572 wake_up_all(&glink->tx_avail_notify); 1573 1574 spin_lock_irqsave(&glink->idr_lock, flags); 1575 channel = idr_find(&glink->lcids, lcid); 1576 if (WARN(!channel, "close ack on unknown channel\n")) { 1577 spin_unlock_irqrestore(&glink->idr_lock, flags); 1578 return; 1579 } 1580 1581 idr_remove(&glink->lcids, channel->lcid); 1582 channel->lcid = 0; 1583 spin_unlock_irqrestore(&glink->idr_lock, flags); 1584 1585 /* Decouple the potential rpdev from the channel */ 1586 if (channel->rpdev) { 1587 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1588 chinfo.src = RPMSG_ADDR_ANY; 1589 chinfo.dst = RPMSG_ADDR_ANY; 1590 1591 rpmsg_unregister_device(glink->dev, &chinfo); 1592 } 1593 channel->rpdev = NULL; 1594 1595 kref_put(&channel->refcount, qcom_glink_channel_release); 1596 } 1597 1598 static void qcom_glink_work(struct work_struct *work) 1599 { 1600 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1601 rx_work); 1602 struct glink_defer_cmd *dcmd; 1603 struct glink_msg *msg; 1604 unsigned long flags; 1605 unsigned int param1; 1606 unsigned int param2; 1607 unsigned int cmd; 1608 1609 for (;;) { 1610 spin_lock_irqsave(&glink->rx_lock, flags); 1611 if (list_empty(&glink->rx_queue)) { 1612 spin_unlock_irqrestore(&glink->rx_lock, flags); 1613 break; 1614 } 1615 dcmd = list_first_entry(&glink->rx_queue, 1616 struct glink_defer_cmd, node); 1617 list_del(&dcmd->node); 1618 spin_unlock_irqrestore(&glink->rx_lock, flags); 1619 1620 msg = &dcmd->msg; 1621 cmd = le16_to_cpu(msg->cmd); 1622 param1 = le16_to_cpu(msg->param1); 1623 param2 = le32_to_cpu(msg->param2); 1624 1625 switch (cmd) { 1626 case RPM_CMD_VERSION: 1627 qcom_glink_receive_version(glink, param1, param2); 1628 break; 1629 case RPM_CMD_VERSION_ACK: 1630 qcom_glink_receive_version_ack(glink, param1, param2); 1631 break; 1632 case RPM_CMD_OPEN: 1633 qcom_glink_rx_open(glink, param1, msg->data); 1634 break; 1635 case RPM_CMD_CLOSE: 1636 qcom_glink_rx_close(glink, param1); 1637 break; 1638 case RPM_CMD_CLOSE_ACK: 1639 qcom_glink_rx_close_ack(glink, param1); 1640 break; 1641 case RPM_CMD_RX_INTENT_REQ: 1642 qcom_glink_handle_intent_req(glink, param1, param2); 1643 break; 1644 default: 1645 WARN(1, "Unknown defer object %d\n", cmd); 1646 break; 1647 } 1648 1649 kfree(dcmd); 1650 } 1651 } 1652 1653 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1654 { 1655 struct glink_defer_cmd *dcmd; 1656 struct glink_defer_cmd *tmp; 1657 1658 /* cancel any pending deferred rx_work */ 1659 cancel_work_sync(&glink->rx_work); 1660 1661 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1662 kfree(dcmd); 1663 } 1664 1665 static ssize_t rpmsg_name_show(struct device *dev, 1666 struct device_attribute *attr, char *buf) 1667 { 1668 int ret = 0; 1669 const char *name; 1670 1671 ret = of_property_read_string(dev->of_node, "label", &name); 1672 if (ret < 0) 1673 name = dev->of_node->name; 1674 1675 return sysfs_emit(buf, "%s\n", name); 1676 } 1677 static DEVICE_ATTR_RO(rpmsg_name); 1678 1679 static struct attribute *qcom_glink_attrs[] = { 1680 &dev_attr_rpmsg_name.attr, 1681 NULL 1682 }; 1683 ATTRIBUTE_GROUPS(qcom_glink); 1684 1685 static void qcom_glink_device_release(struct device *dev) 1686 { 1687 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1688 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1689 1690 /* Release qcom_glink_alloc_channel() reference */ 1691 kref_put(&channel->refcount, qcom_glink_channel_release); 1692 kfree(rpdev); 1693 } 1694 1695 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1696 { 1697 struct rpmsg_device *rpdev; 1698 struct glink_channel *channel; 1699 1700 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1701 if (!rpdev) 1702 return -ENOMEM; 1703 1704 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1705 if (IS_ERR(channel)) { 1706 kfree(rpdev); 1707 return PTR_ERR(channel); 1708 } 1709 channel->rpdev = rpdev; 1710 1711 rpdev->ept = &channel->ept; 1712 rpdev->ops = &glink_device_ops; 1713 rpdev->dev.parent = glink->dev; 1714 rpdev->dev.release = qcom_glink_device_release; 1715 1716 return rpmsg_ctrldev_register_device(rpdev); 1717 } 1718 1719 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1720 unsigned long features, 1721 struct qcom_glink_pipe *rx, 1722 struct qcom_glink_pipe *tx, 1723 bool intentless) 1724 { 1725 int irq; 1726 int ret; 1727 struct qcom_glink *glink; 1728 1729 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1730 if (!glink) 1731 return ERR_PTR(-ENOMEM); 1732 1733 glink->dev = dev; 1734 glink->tx_pipe = tx; 1735 glink->rx_pipe = rx; 1736 1737 glink->features = features; 1738 glink->intentless = intentless; 1739 1740 spin_lock_init(&glink->tx_lock); 1741 spin_lock_init(&glink->rx_lock); 1742 INIT_LIST_HEAD(&glink->rx_queue); 1743 INIT_WORK(&glink->rx_work, qcom_glink_work); 1744 init_waitqueue_head(&glink->tx_avail_notify); 1745 1746 spin_lock_init(&glink->idr_lock); 1747 idr_init(&glink->lcids); 1748 idr_init(&glink->rcids); 1749 1750 glink->dev->groups = qcom_glink_groups; 1751 1752 ret = device_add_groups(dev, qcom_glink_groups); 1753 if (ret) 1754 dev_err(dev, "failed to add groups\n"); 1755 1756 glink->mbox_client.dev = dev; 1757 glink->mbox_client.knows_txdone = true; 1758 glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0); 1759 if (IS_ERR(glink->mbox_chan)) { 1760 if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER) 1761 dev_err(dev, "failed to acquire IPC channel\n"); 1762 return ERR_CAST(glink->mbox_chan); 1763 } 1764 1765 irq = of_irq_get(dev->of_node, 0); 1766 ret = devm_request_irq(dev, irq, 1767 qcom_glink_native_intr, 1768 IRQF_NO_SUSPEND | IRQF_SHARED, 1769 "glink-native", glink); 1770 if (ret) { 1771 dev_err(dev, "failed to request IRQ\n"); 1772 return ERR_PTR(ret); 1773 } 1774 1775 glink->irq = irq; 1776 1777 ret = qcom_glink_send_version(glink); 1778 if (ret) 1779 return ERR_PTR(ret); 1780 1781 ret = qcom_glink_create_chrdev(glink); 1782 if (ret) 1783 dev_err(glink->dev, "failed to register chrdev\n"); 1784 1785 return glink; 1786 } 1787 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1788 1789 static int qcom_glink_remove_device(struct device *dev, void *data) 1790 { 1791 device_unregister(dev); 1792 1793 return 0; 1794 } 1795 1796 void qcom_glink_native_remove(struct qcom_glink *glink) 1797 { 1798 struct glink_channel *channel; 1799 int cid; 1800 int ret; 1801 1802 disable_irq(glink->irq); 1803 qcom_glink_cancel_rx_work(glink); 1804 1805 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1806 if (ret) 1807 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1808 1809 /* Release any defunct local channels, waiting for close-ack */ 1810 idr_for_each_entry(&glink->lcids, channel, cid) 1811 kref_put(&channel->refcount, qcom_glink_channel_release); 1812 1813 /* Release any defunct local channels, waiting for close-req */ 1814 idr_for_each_entry(&glink->rcids, channel, cid) 1815 kref_put(&channel->refcount, qcom_glink_channel_release); 1816 1817 idr_destroy(&glink->lcids); 1818 idr_destroy(&glink->rcids); 1819 mbox_free_channel(glink->mbox_chan); 1820 } 1821 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1822 1823 void qcom_glink_native_unregister(struct qcom_glink *glink) 1824 { 1825 device_unregister(glink->dev); 1826 } 1827 EXPORT_SYMBOL_GPL(qcom_glink_native_unregister); 1828 1829 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1830 MODULE_LICENSE("GPL v2"); 1831