1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/platform_device.h> 15 #include <linux/regmap.h> 16 #include <linux/rpmsg.h> 17 #include <linux/sizes.h> 18 #include <linux/slab.h> 19 #include <linux/wait.h> 20 #include <linux/workqueue.h> 21 #include <linux/mailbox_client.h> 22 23 #include "rpmsg_internal.h" 24 #include "qcom_glink_native.h" 25 26 #define GLINK_NAME_SIZE 32 27 #define GLINK_VERSION_1 1 28 29 #define RPM_GLINK_CID_MIN 1 30 #define RPM_GLINK_CID_MAX 65536 31 32 struct glink_msg { 33 __le16 cmd; 34 __le16 param1; 35 __le32 param2; 36 u8 data[]; 37 } __packed; 38 39 /** 40 * struct glink_defer_cmd - deferred incoming control message 41 * @node: list node 42 * @msg: message header 43 * @data: payload of the message 44 * 45 * Copy of a received control message, to be added to @rx_queue and processed 46 * by @rx_work of @qcom_glink. 47 */ 48 struct glink_defer_cmd { 49 struct list_head node; 50 51 struct glink_msg msg; 52 u8 data[]; 53 }; 54 55 /** 56 * struct glink_core_rx_intent - RX intent 57 * RX intent 58 * 59 * @data: pointer to the data (may be NULL for zero-copy) 60 * @id: remote or local intent ID 61 * @size: size of the original intent (do not modify) 62 * @reuse: To mark if the intent can be reused after first use 63 * @in_use: To mark if intent is already in use for the channel 64 * @offset: next write offset (initially 0) 65 * @node: list node 66 */ 67 struct glink_core_rx_intent { 68 void *data; 69 u32 id; 70 size_t size; 71 bool reuse; 72 bool in_use; 73 u32 offset; 74 75 struct list_head node; 76 }; 77 78 /** 79 * struct qcom_glink - driver context, relates to one remote subsystem 80 * @dev: reference to the associated struct device 81 * @rx_pipe: pipe object for receive FIFO 82 * @tx_pipe: pipe object for transmit FIFO 83 * @rx_work: worker for handling received control messages 84 * @rx_lock: protects the @rx_queue 85 * @rx_queue: queue of received control messages to be processed in @rx_work 86 * @tx_lock: synchronizes operations on the tx fifo 87 * @idr_lock: synchronizes @lcids and @rcids modifications 88 * @lcids: idr of all channels with a known local channel id 89 * @rcids: idr of all channels with a known remote channel id 90 * @features: remote features 91 * @intentless: flag to indicate that there is no intent 92 * @tx_avail_notify: Waitqueue for pending tx tasks 93 * @sent_read_notify: flag to check cmd sent or not 94 * @abort_tx: flag indicating that all tx attempts should fail 95 */ 96 struct qcom_glink { 97 struct device *dev; 98 99 struct qcom_glink_pipe *rx_pipe; 100 struct qcom_glink_pipe *tx_pipe; 101 102 struct work_struct rx_work; 103 spinlock_t rx_lock; 104 struct list_head rx_queue; 105 106 spinlock_t tx_lock; 107 108 spinlock_t idr_lock; 109 struct idr lcids; 110 struct idr rcids; 111 unsigned long features; 112 113 bool intentless; 114 wait_queue_head_t tx_avail_notify; 115 bool sent_read_notify; 116 117 bool abort_tx; 118 }; 119 120 enum { 121 GLINK_STATE_CLOSED, 122 GLINK_STATE_OPENING, 123 GLINK_STATE_OPEN, 124 GLINK_STATE_CLOSING, 125 }; 126 127 /** 128 * struct glink_channel - internal representation of a channel 129 * @rpdev: rpdev reference, only used for primary endpoints 130 * @ept: rpmsg endpoint this channel is associated with 131 * @glink: qcom_glink context handle 132 * @refcount: refcount for the channel object 133 * @recv_lock: guard for @ept.cb 134 * @name: unique channel name/identifier 135 * @lcid: channel id, in local space 136 * @rcid: channel id, in remote space 137 * @intent_lock: lock for protection of @liids, @riids 138 * @liids: idr of all local intents 139 * @riids: idr of all remote intents 140 * @intent_work: worker responsible for transmitting rx_done packets 141 * @done_intents: list of intents that needs to be announced rx_done 142 * @buf: receive buffer, for gathering fragments 143 * @buf_offset: write offset in @buf 144 * @buf_size: size of current @buf 145 * @open_ack: completed once remote has acked the open-request 146 * @open_req: completed once open-request has been received 147 * @intent_req_lock: Synchronises multiple intent requests 148 * @intent_req_result: Result of intent request 149 * @intent_received: flag indicating that an intent has been received 150 * @intent_req_wq: wait queue for intent_req signalling 151 */ 152 struct glink_channel { 153 struct rpmsg_endpoint ept; 154 155 struct rpmsg_device *rpdev; 156 struct qcom_glink *glink; 157 158 struct kref refcount; 159 160 spinlock_t recv_lock; 161 162 char *name; 163 unsigned int lcid; 164 unsigned int rcid; 165 166 spinlock_t intent_lock; 167 struct idr liids; 168 struct idr riids; 169 struct work_struct intent_work; 170 struct list_head done_intents; 171 172 struct glink_core_rx_intent *buf; 173 int buf_offset; 174 int buf_size; 175 176 struct completion open_ack; 177 struct completion open_req; 178 179 struct mutex intent_req_lock; 180 int intent_req_result; 181 bool intent_received; 182 wait_queue_head_t intent_req_wq; 183 }; 184 185 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 186 187 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 188 189 #define GLINK_CMD_VERSION 0 190 #define GLINK_CMD_VERSION_ACK 1 191 #define GLINK_CMD_OPEN 2 192 #define GLINK_CMD_CLOSE 3 193 #define GLINK_CMD_OPEN_ACK 4 194 #define GLINK_CMD_INTENT 5 195 #define GLINK_CMD_RX_DONE 6 196 #define GLINK_CMD_RX_INTENT_REQ 7 197 #define GLINK_CMD_RX_INTENT_REQ_ACK 8 198 #define GLINK_CMD_TX_DATA 9 199 #define GLINK_CMD_CLOSE_ACK 11 200 #define GLINK_CMD_TX_DATA_CONT 12 201 #define GLINK_CMD_READ_NOTIF 13 202 #define GLINK_CMD_RX_DONE_W_REUSE 14 203 204 #define GLINK_FEATURE_INTENTLESS BIT(1) 205 206 static void qcom_glink_rx_done_work(struct work_struct *work); 207 208 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 209 const char *name) 210 { 211 struct glink_channel *channel; 212 213 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 214 if (!channel) 215 return ERR_PTR(-ENOMEM); 216 217 /* Setup glink internal glink_channel data */ 218 spin_lock_init(&channel->recv_lock); 219 spin_lock_init(&channel->intent_lock); 220 mutex_init(&channel->intent_req_lock); 221 222 channel->glink = glink; 223 channel->name = kstrdup(name, GFP_KERNEL); 224 225 init_completion(&channel->open_req); 226 init_completion(&channel->open_ack); 227 init_waitqueue_head(&channel->intent_req_wq); 228 229 INIT_LIST_HEAD(&channel->done_intents); 230 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 231 232 idr_init(&channel->liids); 233 idr_init(&channel->riids); 234 kref_init(&channel->refcount); 235 236 return channel; 237 } 238 239 static void qcom_glink_channel_release(struct kref *ref) 240 { 241 struct glink_channel *channel = container_of(ref, struct glink_channel, 242 refcount); 243 struct glink_core_rx_intent *intent; 244 struct glink_core_rx_intent *tmp; 245 unsigned long flags; 246 int iid; 247 248 /* cancel pending rx_done work */ 249 cancel_work_sync(&channel->intent_work); 250 251 spin_lock_irqsave(&channel->intent_lock, flags); 252 /* Free all non-reuse intents pending rx_done work */ 253 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 254 if (!intent->reuse) { 255 kfree(intent->data); 256 kfree(intent); 257 } 258 } 259 260 idr_for_each_entry(&channel->liids, tmp, iid) { 261 kfree(tmp->data); 262 kfree(tmp); 263 } 264 idr_destroy(&channel->liids); 265 266 idr_for_each_entry(&channel->riids, tmp, iid) 267 kfree(tmp); 268 idr_destroy(&channel->riids); 269 spin_unlock_irqrestore(&channel->intent_lock, flags); 270 271 kfree(channel->name); 272 kfree(channel); 273 } 274 275 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 276 { 277 return glink->rx_pipe->avail(glink->rx_pipe); 278 } 279 280 static void qcom_glink_rx_peek(struct qcom_glink *glink, 281 void *data, unsigned int offset, size_t count) 282 { 283 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count); 284 } 285 286 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 287 { 288 glink->rx_pipe->advance(glink->rx_pipe, count); 289 } 290 291 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 292 { 293 return glink->tx_pipe->avail(glink->tx_pipe); 294 } 295 296 static void qcom_glink_tx_write(struct qcom_glink *glink, 297 const void *hdr, size_t hlen, 298 const void *data, size_t dlen) 299 { 300 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 301 } 302 303 static void qcom_glink_tx_kick(struct qcom_glink *glink) 304 { 305 glink->tx_pipe->kick(glink->tx_pipe); 306 } 307 308 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 309 { 310 struct glink_msg msg; 311 312 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF); 313 msg.param1 = 0; 314 msg.param2 = 0; 315 316 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 317 318 qcom_glink_tx_kick(glink); 319 } 320 321 static int qcom_glink_tx(struct qcom_glink *glink, 322 const void *hdr, size_t hlen, 323 const void *data, size_t dlen, bool wait) 324 { 325 unsigned int tlen = hlen + dlen; 326 unsigned long flags; 327 int ret = 0; 328 329 /* Reject packets that are too big */ 330 if (tlen >= glink->tx_pipe->length) 331 return -EINVAL; 332 333 spin_lock_irqsave(&glink->tx_lock, flags); 334 335 if (glink->abort_tx) { 336 ret = -EIO; 337 goto out; 338 } 339 340 while (qcom_glink_tx_avail(glink) < tlen) { 341 if (!wait) { 342 ret = -EAGAIN; 343 goto out; 344 } 345 346 if (glink->abort_tx) { 347 ret = -EIO; 348 goto out; 349 } 350 351 if (!glink->sent_read_notify) { 352 glink->sent_read_notify = true; 353 qcom_glink_send_read_notify(glink); 354 } 355 356 /* Wait without holding the tx_lock */ 357 spin_unlock_irqrestore(&glink->tx_lock, flags); 358 359 wait_event_timeout(glink->tx_avail_notify, 360 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 361 362 spin_lock_irqsave(&glink->tx_lock, flags); 363 364 if (qcom_glink_tx_avail(glink) >= tlen) 365 glink->sent_read_notify = false; 366 } 367 368 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 369 qcom_glink_tx_kick(glink); 370 371 out: 372 spin_unlock_irqrestore(&glink->tx_lock, flags); 373 374 return ret; 375 } 376 377 static int qcom_glink_send_version(struct qcom_glink *glink) 378 { 379 struct glink_msg msg; 380 381 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION); 382 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 383 msg.param2 = cpu_to_le32(glink->features); 384 385 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 386 } 387 388 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 389 { 390 struct glink_msg msg; 391 392 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK); 393 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 394 msg.param2 = cpu_to_le32(glink->features); 395 396 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 397 } 398 399 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 400 struct glink_channel *channel) 401 { 402 struct glink_msg msg; 403 404 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK); 405 msg.param1 = cpu_to_le16(channel->rcid); 406 msg.param2 = cpu_to_le32(0); 407 408 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 409 } 410 411 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 412 unsigned int cid, bool granted) 413 { 414 struct glink_channel *channel; 415 unsigned long flags; 416 417 spin_lock_irqsave(&glink->idr_lock, flags); 418 channel = idr_find(&glink->rcids, cid); 419 spin_unlock_irqrestore(&glink->idr_lock, flags); 420 if (!channel) { 421 dev_err(glink->dev, "unable to find channel\n"); 422 return; 423 } 424 425 WRITE_ONCE(channel->intent_req_result, granted); 426 wake_up_all(&channel->intent_req_wq); 427 } 428 429 static void qcom_glink_intent_req_abort(struct glink_channel *channel) 430 { 431 WRITE_ONCE(channel->intent_req_result, 0); 432 wake_up_all(&channel->intent_req_wq); 433 } 434 435 /** 436 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote 437 * @glink: Ptr to the glink edge 438 * @channel: Ptr to the channel that the open req is sent 439 * 440 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote. 441 * Will return with refcount held, regardless of outcome. 442 * 443 * Return: 0 on success, negative errno otherwise. 444 */ 445 static int qcom_glink_send_open_req(struct qcom_glink *glink, 446 struct glink_channel *channel) 447 { 448 struct { 449 struct glink_msg msg; 450 u8 name[GLINK_NAME_SIZE]; 451 } __packed req; 452 int name_len = strlen(channel->name) + 1; 453 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 454 int ret; 455 unsigned long flags; 456 457 kref_get(&channel->refcount); 458 459 spin_lock_irqsave(&glink->idr_lock, flags); 460 ret = idr_alloc_cyclic(&glink->lcids, channel, 461 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 462 GFP_ATOMIC); 463 spin_unlock_irqrestore(&glink->idr_lock, flags); 464 if (ret < 0) 465 return ret; 466 467 channel->lcid = ret; 468 469 req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN); 470 req.msg.param1 = cpu_to_le16(channel->lcid); 471 req.msg.param2 = cpu_to_le32(name_len); 472 strcpy(req.name, channel->name); 473 474 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 475 if (ret) 476 goto remove_idr; 477 478 return 0; 479 480 remove_idr: 481 spin_lock_irqsave(&glink->idr_lock, flags); 482 idr_remove(&glink->lcids, channel->lcid); 483 channel->lcid = 0; 484 spin_unlock_irqrestore(&glink->idr_lock, flags); 485 486 return ret; 487 } 488 489 static void qcom_glink_send_close_req(struct qcom_glink *glink, 490 struct glink_channel *channel) 491 { 492 struct glink_msg req; 493 494 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE); 495 req.param1 = cpu_to_le16(channel->lcid); 496 req.param2 = 0; 497 498 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 499 } 500 501 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 502 unsigned int rcid) 503 { 504 struct glink_msg req; 505 506 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK); 507 req.param1 = cpu_to_le16(rcid); 508 req.param2 = 0; 509 510 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 511 } 512 513 static void qcom_glink_rx_done_work(struct work_struct *work) 514 { 515 struct glink_channel *channel = container_of(work, struct glink_channel, 516 intent_work); 517 struct qcom_glink *glink = channel->glink; 518 struct glink_core_rx_intent *intent, *tmp; 519 struct { 520 u16 id; 521 u16 lcid; 522 u32 liid; 523 } __packed cmd; 524 525 unsigned int cid = channel->lcid; 526 unsigned int iid; 527 bool reuse; 528 unsigned long flags; 529 530 spin_lock_irqsave(&channel->intent_lock, flags); 531 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 532 list_del(&intent->node); 533 spin_unlock_irqrestore(&channel->intent_lock, flags); 534 iid = intent->id; 535 reuse = intent->reuse; 536 537 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE; 538 cmd.lcid = cid; 539 cmd.liid = iid; 540 541 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 542 if (!reuse) { 543 kfree(intent->data); 544 kfree(intent); 545 } 546 spin_lock_irqsave(&channel->intent_lock, flags); 547 } 548 spin_unlock_irqrestore(&channel->intent_lock, flags); 549 } 550 551 static void qcom_glink_rx_done(struct qcom_glink *glink, 552 struct glink_channel *channel, 553 struct glink_core_rx_intent *intent) 554 { 555 /* We don't send RX_DONE to intentless systems */ 556 if (glink->intentless) { 557 kfree(intent->data); 558 kfree(intent); 559 return; 560 } 561 562 /* Take it off the tree of receive intents */ 563 if (!intent->reuse) { 564 spin_lock(&channel->intent_lock); 565 idr_remove(&channel->liids, intent->id); 566 spin_unlock(&channel->intent_lock); 567 } 568 569 /* Schedule the sending of a rx_done indication */ 570 spin_lock(&channel->intent_lock); 571 list_add_tail(&intent->node, &channel->done_intents); 572 spin_unlock(&channel->intent_lock); 573 574 schedule_work(&channel->intent_work); 575 } 576 577 /** 578 * qcom_glink_receive_version() - receive version/features from remote system 579 * 580 * @glink: pointer to transport interface 581 * @version: remote version 582 * @features: remote features 583 * 584 * This function is called in response to a remote-initiated version/feature 585 * negotiation sequence. 586 */ 587 static void qcom_glink_receive_version(struct qcom_glink *glink, 588 u32 version, 589 u32 features) 590 { 591 switch (version) { 592 case 0: 593 break; 594 case GLINK_VERSION_1: 595 glink->features &= features; 596 fallthrough; 597 default: 598 qcom_glink_send_version_ack(glink); 599 break; 600 } 601 } 602 603 /** 604 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 605 * 606 * @glink: pointer to transport interface 607 * @version: remote version response 608 * @features: remote features response 609 * 610 * This function is called in response to a local-initiated version/feature 611 * negotiation sequence and is the counter-offer from the remote side based 612 * upon the initial version and feature set requested. 613 */ 614 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 615 u32 version, 616 u32 features) 617 { 618 switch (version) { 619 case 0: 620 /* Version negotiation failed */ 621 break; 622 case GLINK_VERSION_1: 623 if (features == glink->features) 624 break; 625 626 glink->features &= features; 627 fallthrough; 628 default: 629 qcom_glink_send_version(glink); 630 break; 631 } 632 } 633 634 /** 635 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 636 * wire format and transmit 637 * @glink: The transport to transmit on. 638 * @channel: The glink channel 639 * @granted: The request response to encode. 640 * 641 * Return: 0 on success or standard Linux error code. 642 */ 643 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 644 struct glink_channel *channel, 645 bool granted) 646 { 647 struct glink_msg msg; 648 649 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK); 650 msg.param1 = cpu_to_le16(channel->lcid); 651 msg.param2 = cpu_to_le32(granted); 652 653 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 654 655 return 0; 656 } 657 658 /** 659 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 660 * transmit 661 * @glink: The transport to transmit on. 662 * @channel: The local channel 663 * @intent: The intent to pass on to remote. 664 * 665 * Return: 0 on success or standard Linux error code. 666 */ 667 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 668 struct glink_channel *channel, 669 struct glink_core_rx_intent *intent) 670 { 671 struct command { 672 __le16 id; 673 __le16 lcid; 674 __le32 count; 675 __le32 size; 676 __le32 liid; 677 } __packed; 678 struct command cmd; 679 680 cmd.id = cpu_to_le16(GLINK_CMD_INTENT); 681 cmd.lcid = cpu_to_le16(channel->lcid); 682 cmd.count = cpu_to_le32(1); 683 cmd.size = cpu_to_le32(intent->size); 684 cmd.liid = cpu_to_le32(intent->id); 685 686 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 687 688 return 0; 689 } 690 691 static struct glink_core_rx_intent * 692 qcom_glink_alloc_intent(struct qcom_glink *glink, 693 struct glink_channel *channel, 694 size_t size, 695 bool reuseable) 696 { 697 struct glink_core_rx_intent *intent; 698 int ret; 699 unsigned long flags; 700 701 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 702 if (!intent) 703 return NULL; 704 705 intent->data = kzalloc(size, GFP_KERNEL); 706 if (!intent->data) 707 goto free_intent; 708 709 spin_lock_irqsave(&channel->intent_lock, flags); 710 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 711 if (ret < 0) { 712 spin_unlock_irqrestore(&channel->intent_lock, flags); 713 goto free_data; 714 } 715 spin_unlock_irqrestore(&channel->intent_lock, flags); 716 717 intent->id = ret; 718 intent->size = size; 719 intent->reuse = reuseable; 720 721 return intent; 722 723 free_data: 724 kfree(intent->data); 725 free_intent: 726 kfree(intent); 727 return NULL; 728 } 729 730 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 731 u32 cid, uint32_t iid, 732 bool reuse) 733 { 734 struct glink_core_rx_intent *intent; 735 struct glink_channel *channel; 736 unsigned long flags; 737 738 spin_lock_irqsave(&glink->idr_lock, flags); 739 channel = idr_find(&glink->rcids, cid); 740 spin_unlock_irqrestore(&glink->idr_lock, flags); 741 if (!channel) { 742 dev_err(glink->dev, "invalid channel id received\n"); 743 return; 744 } 745 746 spin_lock_irqsave(&channel->intent_lock, flags); 747 intent = idr_find(&channel->riids, iid); 748 749 if (!intent) { 750 spin_unlock_irqrestore(&channel->intent_lock, flags); 751 dev_err(glink->dev, "invalid intent id received\n"); 752 return; 753 } 754 755 intent->in_use = false; 756 757 if (!reuse) { 758 idr_remove(&channel->riids, intent->id); 759 kfree(intent); 760 } 761 spin_unlock_irqrestore(&channel->intent_lock, flags); 762 763 if (reuse) { 764 WRITE_ONCE(channel->intent_received, true); 765 wake_up_all(&channel->intent_req_wq); 766 } 767 } 768 769 /** 770 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 771 * from remote side 772 * @glink: Pointer to the transport interface 773 * @cid: Remote channel ID 774 * @size: size of the intent 775 * 776 * The function searches for the local channel to which the request for 777 * rx_intent has arrived and allocates and notifies the remote back 778 */ 779 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 780 u32 cid, size_t size) 781 { 782 struct glink_core_rx_intent *intent; 783 struct glink_channel *channel; 784 unsigned long flags; 785 786 spin_lock_irqsave(&glink->idr_lock, flags); 787 channel = idr_find(&glink->rcids, cid); 788 spin_unlock_irqrestore(&glink->idr_lock, flags); 789 790 if (!channel) { 791 pr_err("%s channel not found for cid %d\n", __func__, cid); 792 return; 793 } 794 795 intent = qcom_glink_alloc_intent(glink, channel, size, false); 796 if (intent) 797 qcom_glink_advertise_intent(glink, channel, intent); 798 799 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 800 } 801 802 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 803 { 804 struct glink_defer_cmd *dcmd; 805 806 extra = ALIGN(extra, 8); 807 808 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 809 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 810 return -ENXIO; 811 } 812 813 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 814 if (!dcmd) 815 return -ENOMEM; 816 817 INIT_LIST_HEAD(&dcmd->node); 818 819 qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 820 821 spin_lock(&glink->rx_lock); 822 list_add_tail(&dcmd->node, &glink->rx_queue); 823 spin_unlock(&glink->rx_lock); 824 825 schedule_work(&glink->rx_work); 826 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 827 828 return 0; 829 } 830 831 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 832 { 833 struct glink_core_rx_intent *intent; 834 struct glink_channel *channel; 835 struct { 836 struct glink_msg msg; 837 __le32 chunk_size; 838 __le32 left_size; 839 } __packed hdr; 840 unsigned int chunk_size; 841 unsigned int left_size; 842 unsigned int rcid; 843 unsigned int liid; 844 int ret = 0; 845 unsigned long flags; 846 847 if (avail < sizeof(hdr)) { 848 dev_dbg(glink->dev, "Not enough data in fifo\n"); 849 return -EAGAIN; 850 } 851 852 qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr)); 853 chunk_size = le32_to_cpu(hdr.chunk_size); 854 left_size = le32_to_cpu(hdr.left_size); 855 856 if (avail < sizeof(hdr) + chunk_size) { 857 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 858 return -EAGAIN; 859 } 860 861 rcid = le16_to_cpu(hdr.msg.param1); 862 spin_lock_irqsave(&glink->idr_lock, flags); 863 channel = idr_find(&glink->rcids, rcid); 864 spin_unlock_irqrestore(&glink->idr_lock, flags); 865 if (!channel) { 866 dev_dbg(glink->dev, "Data on non-existing channel\n"); 867 868 /* Drop the message */ 869 goto advance_rx; 870 } 871 872 if (glink->intentless) { 873 /* Might have an ongoing, fragmented, message to append */ 874 if (!channel->buf) { 875 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 876 if (!intent) 877 return -ENOMEM; 878 879 intent->data = kmalloc(chunk_size + left_size, 880 GFP_ATOMIC); 881 if (!intent->data) { 882 kfree(intent); 883 return -ENOMEM; 884 } 885 886 intent->id = 0xbabababa; 887 intent->size = chunk_size + left_size; 888 intent->offset = 0; 889 890 channel->buf = intent; 891 } else { 892 intent = channel->buf; 893 } 894 } else { 895 liid = le32_to_cpu(hdr.msg.param2); 896 897 spin_lock_irqsave(&channel->intent_lock, flags); 898 intent = idr_find(&channel->liids, liid); 899 spin_unlock_irqrestore(&channel->intent_lock, flags); 900 901 if (!intent) { 902 dev_err(glink->dev, 903 "no intent found for channel %s intent %d", 904 channel->name, liid); 905 ret = -ENOENT; 906 goto advance_rx; 907 } 908 } 909 910 if (intent->size - intent->offset < chunk_size) { 911 dev_err(glink->dev, "Insufficient space in intent\n"); 912 913 /* The packet header lied, drop payload */ 914 goto advance_rx; 915 } 916 917 qcom_glink_rx_peek(glink, intent->data + intent->offset, 918 sizeof(hdr), chunk_size); 919 intent->offset += chunk_size; 920 921 /* Handle message when no fragments remain to be received */ 922 if (!left_size) { 923 spin_lock(&channel->recv_lock); 924 if (channel->ept.cb) { 925 channel->ept.cb(channel->ept.rpdev, 926 intent->data, 927 intent->offset, 928 channel->ept.priv, 929 RPMSG_ADDR_ANY); 930 } 931 spin_unlock(&channel->recv_lock); 932 933 intent->offset = 0; 934 channel->buf = NULL; 935 936 qcom_glink_rx_done(glink, channel, intent); 937 } 938 939 advance_rx: 940 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 941 942 return ret; 943 } 944 945 static void qcom_glink_handle_intent(struct qcom_glink *glink, 946 unsigned int cid, 947 unsigned int count, 948 size_t avail) 949 { 950 struct glink_core_rx_intent *intent; 951 struct glink_channel *channel; 952 struct intent_pair { 953 __le32 size; 954 __le32 iid; 955 }; 956 957 struct { 958 struct glink_msg msg; 959 struct intent_pair intents[]; 960 } __packed * msg; 961 962 const size_t msglen = struct_size(msg, intents, count); 963 int ret; 964 int i; 965 unsigned long flags; 966 967 if (avail < msglen) { 968 dev_dbg(glink->dev, "Not enough data in fifo\n"); 969 return; 970 } 971 972 spin_lock_irqsave(&glink->idr_lock, flags); 973 channel = idr_find(&glink->rcids, cid); 974 spin_unlock_irqrestore(&glink->idr_lock, flags); 975 if (!channel) { 976 dev_err(glink->dev, "intents for non-existing channel\n"); 977 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 978 return; 979 } 980 981 msg = kmalloc(msglen, GFP_ATOMIC); 982 if (!msg) 983 return; 984 985 qcom_glink_rx_peek(glink, msg, 0, msglen); 986 987 for (i = 0; i < count; ++i) { 988 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 989 if (!intent) 990 break; 991 992 intent->id = le32_to_cpu(msg->intents[i].iid); 993 intent->size = le32_to_cpu(msg->intents[i].size); 994 995 spin_lock_irqsave(&channel->intent_lock, flags); 996 ret = idr_alloc(&channel->riids, intent, 997 intent->id, intent->id + 1, GFP_ATOMIC); 998 spin_unlock_irqrestore(&channel->intent_lock, flags); 999 1000 if (ret < 0) 1001 dev_err(glink->dev, "failed to store remote intent\n"); 1002 } 1003 1004 WRITE_ONCE(channel->intent_received, true); 1005 wake_up_all(&channel->intent_req_wq); 1006 1007 kfree(msg); 1008 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 1009 } 1010 1011 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 1012 { 1013 struct glink_channel *channel; 1014 1015 spin_lock(&glink->idr_lock); 1016 channel = idr_find(&glink->lcids, lcid); 1017 spin_unlock(&glink->idr_lock); 1018 if (!channel) { 1019 dev_err(glink->dev, "Invalid open ack packet\n"); 1020 return -EINVAL; 1021 } 1022 1023 complete_all(&channel->open_ack); 1024 1025 return 0; 1026 } 1027 1028 void qcom_glink_native_rx(struct qcom_glink *glink) 1029 { 1030 struct glink_msg msg; 1031 unsigned int param1; 1032 unsigned int param2; 1033 unsigned int avail; 1034 unsigned int cmd; 1035 int ret = 0; 1036 1037 /* To wakeup any blocking writers */ 1038 wake_up_all(&glink->tx_avail_notify); 1039 1040 for (;;) { 1041 avail = qcom_glink_rx_avail(glink); 1042 if (avail < sizeof(msg)) 1043 break; 1044 1045 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg)); 1046 1047 cmd = le16_to_cpu(msg.cmd); 1048 param1 = le16_to_cpu(msg.param1); 1049 param2 = le32_to_cpu(msg.param2); 1050 1051 switch (cmd) { 1052 case GLINK_CMD_VERSION: 1053 case GLINK_CMD_VERSION_ACK: 1054 case GLINK_CMD_CLOSE: 1055 case GLINK_CMD_CLOSE_ACK: 1056 case GLINK_CMD_RX_INTENT_REQ: 1057 ret = qcom_glink_rx_defer(glink, 0); 1058 break; 1059 case GLINK_CMD_OPEN_ACK: 1060 ret = qcom_glink_rx_open_ack(glink, param1); 1061 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1062 break; 1063 case GLINK_CMD_OPEN: 1064 ret = qcom_glink_rx_defer(glink, param2); 1065 break; 1066 case GLINK_CMD_TX_DATA: 1067 case GLINK_CMD_TX_DATA_CONT: 1068 ret = qcom_glink_rx_data(glink, avail); 1069 break; 1070 case GLINK_CMD_READ_NOTIF: 1071 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1072 qcom_glink_tx_kick(glink); 1073 break; 1074 case GLINK_CMD_INTENT: 1075 qcom_glink_handle_intent(glink, param1, param2, avail); 1076 break; 1077 case GLINK_CMD_RX_DONE: 1078 qcom_glink_handle_rx_done(glink, param1, param2, false); 1079 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1080 break; 1081 case GLINK_CMD_RX_DONE_W_REUSE: 1082 qcom_glink_handle_rx_done(glink, param1, param2, true); 1083 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1084 break; 1085 case GLINK_CMD_RX_INTENT_REQ_ACK: 1086 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1087 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1088 break; 1089 default: 1090 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1091 ret = -EINVAL; 1092 break; 1093 } 1094 1095 if (ret) 1096 break; 1097 } 1098 } 1099 EXPORT_SYMBOL(qcom_glink_native_rx); 1100 1101 /* Locally initiated rpmsg_create_ept */ 1102 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1103 const char *name) 1104 { 1105 struct glink_channel *channel; 1106 int ret; 1107 unsigned long flags; 1108 1109 channel = qcom_glink_alloc_channel(glink, name); 1110 if (IS_ERR(channel)) 1111 return ERR_CAST(channel); 1112 1113 ret = qcom_glink_send_open_req(glink, channel); 1114 if (ret) 1115 goto release_channel; 1116 1117 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1118 if (!ret) 1119 goto err_timeout; 1120 1121 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1122 if (!ret) 1123 goto err_timeout; 1124 1125 qcom_glink_send_open_ack(glink, channel); 1126 1127 return channel; 1128 1129 err_timeout: 1130 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1131 spin_lock_irqsave(&glink->idr_lock, flags); 1132 idr_remove(&glink->lcids, channel->lcid); 1133 spin_unlock_irqrestore(&glink->idr_lock, flags); 1134 1135 release_channel: 1136 /* Release qcom_glink_send_open_req() reference */ 1137 kref_put(&channel->refcount, qcom_glink_channel_release); 1138 /* Release qcom_glink_alloc_channel() reference */ 1139 kref_put(&channel->refcount, qcom_glink_channel_release); 1140 1141 return ERR_PTR(-ETIMEDOUT); 1142 } 1143 1144 /* Remote initiated rpmsg_create_ept */ 1145 static int qcom_glink_create_remote(struct qcom_glink *glink, 1146 struct glink_channel *channel) 1147 { 1148 int ret; 1149 1150 qcom_glink_send_open_ack(glink, channel); 1151 1152 ret = qcom_glink_send_open_req(glink, channel); 1153 if (ret) 1154 goto close_link; 1155 1156 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1157 if (!ret) { 1158 ret = -ETIMEDOUT; 1159 goto close_link; 1160 } 1161 1162 return 0; 1163 1164 close_link: 1165 /* 1166 * Send a close request to "undo" our open-ack. The close-ack will 1167 * release qcom_glink_send_open_req() reference and the last reference 1168 * will be relesed after receiving remote_close or transport unregister 1169 * by calling qcom_glink_native_remove(). 1170 */ 1171 qcom_glink_send_close_req(glink, channel); 1172 1173 return ret; 1174 } 1175 1176 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1177 rpmsg_rx_cb_t cb, 1178 void *priv, 1179 struct rpmsg_channel_info 1180 chinfo) 1181 { 1182 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1183 struct glink_channel *channel; 1184 struct qcom_glink *glink = parent->glink; 1185 struct rpmsg_endpoint *ept; 1186 const char *name = chinfo.name; 1187 int cid; 1188 int ret; 1189 unsigned long flags; 1190 1191 spin_lock_irqsave(&glink->idr_lock, flags); 1192 idr_for_each_entry(&glink->rcids, channel, cid) { 1193 if (!strcmp(channel->name, name)) 1194 break; 1195 } 1196 spin_unlock_irqrestore(&glink->idr_lock, flags); 1197 1198 if (!channel) { 1199 channel = qcom_glink_create_local(glink, name); 1200 if (IS_ERR(channel)) 1201 return NULL; 1202 } else { 1203 ret = qcom_glink_create_remote(glink, channel); 1204 if (ret) 1205 return NULL; 1206 } 1207 1208 ept = &channel->ept; 1209 ept->rpdev = rpdev; 1210 ept->cb = cb; 1211 ept->priv = priv; 1212 ept->ops = &glink_endpoint_ops; 1213 1214 return ept; 1215 } 1216 1217 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1218 { 1219 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1220 struct device_node *np = rpdev->dev.of_node; 1221 struct qcom_glink *glink = channel->glink; 1222 struct glink_core_rx_intent *intent; 1223 const struct property *prop = NULL; 1224 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1225 int num_intents; 1226 int num_groups = 1; 1227 __be32 *val = defaults; 1228 int size; 1229 1230 if (glink->intentless || !completion_done(&channel->open_ack)) 1231 return 0; 1232 1233 prop = of_find_property(np, "qcom,intents", NULL); 1234 if (prop) { 1235 val = prop->value; 1236 num_groups = prop->length / sizeof(u32) / 2; 1237 } 1238 1239 /* Channel is now open, advertise base set of intents */ 1240 while (num_groups--) { 1241 size = be32_to_cpup(val++); 1242 num_intents = be32_to_cpup(val++); 1243 while (num_intents--) { 1244 intent = qcom_glink_alloc_intent(glink, channel, size, 1245 true); 1246 if (!intent) 1247 break; 1248 1249 qcom_glink_advertise_intent(glink, channel, intent); 1250 } 1251 } 1252 return 0; 1253 } 1254 1255 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1256 { 1257 struct glink_channel *channel = to_glink_channel(ept); 1258 struct qcom_glink *glink = channel->glink; 1259 unsigned long flags; 1260 1261 spin_lock_irqsave(&channel->recv_lock, flags); 1262 channel->ept.cb = NULL; 1263 spin_unlock_irqrestore(&channel->recv_lock, flags); 1264 1265 /* Decouple the potential rpdev from the channel */ 1266 channel->rpdev = NULL; 1267 1268 qcom_glink_send_close_req(glink, channel); 1269 } 1270 1271 static int qcom_glink_request_intent(struct qcom_glink *glink, 1272 struct glink_channel *channel, 1273 size_t size) 1274 { 1275 struct { 1276 u16 id; 1277 u16 cid; 1278 u32 size; 1279 } __packed cmd; 1280 1281 int ret; 1282 1283 mutex_lock(&channel->intent_req_lock); 1284 1285 WRITE_ONCE(channel->intent_req_result, -1); 1286 WRITE_ONCE(channel->intent_received, false); 1287 1288 cmd.id = GLINK_CMD_RX_INTENT_REQ; 1289 cmd.cid = channel->lcid; 1290 cmd.size = size; 1291 1292 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1293 if (ret) 1294 goto unlock; 1295 1296 ret = wait_event_timeout(channel->intent_req_wq, 1297 READ_ONCE(channel->intent_req_result) >= 0 && 1298 READ_ONCE(channel->intent_received), 1299 10 * HZ); 1300 if (!ret) { 1301 dev_err(glink->dev, "intent request timed out\n"); 1302 ret = -ETIMEDOUT; 1303 } else { 1304 ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED; 1305 } 1306 1307 unlock: 1308 mutex_unlock(&channel->intent_req_lock); 1309 return ret; 1310 } 1311 1312 static int __qcom_glink_send(struct glink_channel *channel, 1313 void *data, int len, bool wait) 1314 { 1315 struct qcom_glink *glink = channel->glink; 1316 struct glink_core_rx_intent *intent = NULL; 1317 struct glink_core_rx_intent *tmp; 1318 int iid = 0; 1319 struct { 1320 struct glink_msg msg; 1321 __le32 chunk_size; 1322 __le32 left_size; 1323 } __packed req; 1324 int ret; 1325 unsigned long flags; 1326 int chunk_size = len; 1327 size_t offset = 0; 1328 1329 if (!glink->intentless) { 1330 while (!intent) { 1331 spin_lock_irqsave(&channel->intent_lock, flags); 1332 idr_for_each_entry(&channel->riids, tmp, iid) { 1333 if (tmp->size >= len && !tmp->in_use) { 1334 if (!intent) 1335 intent = tmp; 1336 else if (intent->size > tmp->size) 1337 intent = tmp; 1338 if (intent->size == len) 1339 break; 1340 } 1341 } 1342 if (intent) 1343 intent->in_use = true; 1344 spin_unlock_irqrestore(&channel->intent_lock, flags); 1345 1346 /* We found an available intent */ 1347 if (intent) 1348 break; 1349 1350 if (!wait) 1351 return -EBUSY; 1352 1353 ret = qcom_glink_request_intent(glink, channel, len); 1354 if (ret < 0) 1355 return ret; 1356 } 1357 1358 iid = intent->id; 1359 } 1360 1361 while (offset < len) { 1362 chunk_size = len - offset; 1363 if (chunk_size > SZ_8K && wait) 1364 chunk_size = SZ_8K; 1365 1366 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT); 1367 req.msg.param1 = cpu_to_le16(channel->lcid); 1368 req.msg.param2 = cpu_to_le32(iid); 1369 req.chunk_size = cpu_to_le32(chunk_size); 1370 req.left_size = cpu_to_le32(len - offset - chunk_size); 1371 1372 ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait); 1373 if (ret) { 1374 /* Mark intent available if we failed */ 1375 if (intent) 1376 intent->in_use = false; 1377 return ret; 1378 } 1379 1380 offset += chunk_size; 1381 } 1382 1383 return 0; 1384 } 1385 1386 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1387 { 1388 struct glink_channel *channel = to_glink_channel(ept); 1389 1390 return __qcom_glink_send(channel, data, len, true); 1391 } 1392 1393 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1394 { 1395 struct glink_channel *channel = to_glink_channel(ept); 1396 1397 return __qcom_glink_send(channel, data, len, false); 1398 } 1399 1400 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1401 { 1402 struct glink_channel *channel = to_glink_channel(ept); 1403 1404 return __qcom_glink_send(channel, data, len, true); 1405 } 1406 1407 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1408 { 1409 struct glink_channel *channel = to_glink_channel(ept); 1410 1411 return __qcom_glink_send(channel, data, len, false); 1412 } 1413 1414 /* 1415 * Finds the device_node for the glink child interested in this channel. 1416 */ 1417 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1418 const char *channel) 1419 { 1420 struct device_node *child; 1421 const char *name; 1422 const char *key; 1423 int ret; 1424 1425 for_each_available_child_of_node(node, child) { 1426 key = "qcom,glink-channels"; 1427 ret = of_property_read_string(child, key, &name); 1428 if (ret) 1429 continue; 1430 1431 if (strcmp(name, channel) == 0) 1432 return child; 1433 } 1434 1435 return NULL; 1436 } 1437 1438 static const struct rpmsg_device_ops glink_device_ops = { 1439 .create_ept = qcom_glink_create_ept, 1440 .announce_create = qcom_glink_announce_create, 1441 }; 1442 1443 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1444 .destroy_ept = qcom_glink_destroy_ept, 1445 .send = qcom_glink_send, 1446 .sendto = qcom_glink_sendto, 1447 .trysend = qcom_glink_trysend, 1448 .trysendto = qcom_glink_trysendto, 1449 }; 1450 1451 static void qcom_glink_rpdev_release(struct device *dev) 1452 { 1453 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1454 1455 kfree(rpdev->driver_override); 1456 kfree(rpdev); 1457 } 1458 1459 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1460 char *name) 1461 { 1462 struct glink_channel *channel; 1463 struct rpmsg_device *rpdev; 1464 bool create_device = false; 1465 struct device_node *node; 1466 int lcid; 1467 int ret; 1468 unsigned long flags; 1469 1470 spin_lock_irqsave(&glink->idr_lock, flags); 1471 idr_for_each_entry(&glink->lcids, channel, lcid) { 1472 if (!strcmp(channel->name, name)) 1473 break; 1474 } 1475 spin_unlock_irqrestore(&glink->idr_lock, flags); 1476 1477 if (!channel) { 1478 channel = qcom_glink_alloc_channel(glink, name); 1479 if (IS_ERR(channel)) 1480 return PTR_ERR(channel); 1481 1482 /* The opening dance was initiated by the remote */ 1483 create_device = true; 1484 } 1485 1486 spin_lock_irqsave(&glink->idr_lock, flags); 1487 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1488 if (ret < 0) { 1489 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1490 spin_unlock_irqrestore(&glink->idr_lock, flags); 1491 goto free_channel; 1492 } 1493 channel->rcid = ret; 1494 spin_unlock_irqrestore(&glink->idr_lock, flags); 1495 1496 complete_all(&channel->open_req); 1497 1498 if (create_device) { 1499 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1500 if (!rpdev) { 1501 ret = -ENOMEM; 1502 goto rcid_remove; 1503 } 1504 1505 rpdev->ept = &channel->ept; 1506 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1507 rpdev->src = RPMSG_ADDR_ANY; 1508 rpdev->dst = RPMSG_ADDR_ANY; 1509 rpdev->ops = &glink_device_ops; 1510 1511 node = qcom_glink_match_channel(glink->dev->of_node, name); 1512 rpdev->dev.of_node = node; 1513 rpdev->dev.parent = glink->dev; 1514 rpdev->dev.release = qcom_glink_rpdev_release; 1515 1516 ret = rpmsg_register_device(rpdev); 1517 if (ret) 1518 goto rcid_remove; 1519 1520 channel->rpdev = rpdev; 1521 } 1522 1523 return 0; 1524 1525 rcid_remove: 1526 spin_lock_irqsave(&glink->idr_lock, flags); 1527 idr_remove(&glink->rcids, channel->rcid); 1528 channel->rcid = 0; 1529 spin_unlock_irqrestore(&glink->idr_lock, flags); 1530 free_channel: 1531 /* Release the reference, iff we took it */ 1532 if (create_device) 1533 kref_put(&channel->refcount, qcom_glink_channel_release); 1534 1535 return ret; 1536 } 1537 1538 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1539 { 1540 struct rpmsg_channel_info chinfo; 1541 struct glink_channel *channel; 1542 unsigned long flags; 1543 1544 spin_lock_irqsave(&glink->idr_lock, flags); 1545 channel = idr_find(&glink->rcids, rcid); 1546 spin_unlock_irqrestore(&glink->idr_lock, flags); 1547 if (WARN(!channel, "close request on unknown channel\n")) 1548 return; 1549 1550 /* cancel pending rx_done work */ 1551 cancel_work_sync(&channel->intent_work); 1552 1553 if (channel->rpdev) { 1554 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1555 chinfo.src = RPMSG_ADDR_ANY; 1556 chinfo.dst = RPMSG_ADDR_ANY; 1557 1558 rpmsg_unregister_device(glink->dev, &chinfo); 1559 } 1560 channel->rpdev = NULL; 1561 1562 qcom_glink_send_close_ack(glink, channel->rcid); 1563 1564 spin_lock_irqsave(&glink->idr_lock, flags); 1565 idr_remove(&glink->rcids, channel->rcid); 1566 channel->rcid = 0; 1567 spin_unlock_irqrestore(&glink->idr_lock, flags); 1568 1569 kref_put(&channel->refcount, qcom_glink_channel_release); 1570 } 1571 1572 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1573 { 1574 struct rpmsg_channel_info chinfo; 1575 struct glink_channel *channel; 1576 unsigned long flags; 1577 1578 /* To wakeup any blocking writers */ 1579 wake_up_all(&glink->tx_avail_notify); 1580 1581 spin_lock_irqsave(&glink->idr_lock, flags); 1582 channel = idr_find(&glink->lcids, lcid); 1583 if (WARN(!channel, "close ack on unknown channel\n")) { 1584 spin_unlock_irqrestore(&glink->idr_lock, flags); 1585 return; 1586 } 1587 1588 idr_remove(&glink->lcids, channel->lcid); 1589 channel->lcid = 0; 1590 spin_unlock_irqrestore(&glink->idr_lock, flags); 1591 1592 /* Decouple the potential rpdev from the channel */ 1593 if (channel->rpdev) { 1594 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1595 chinfo.src = RPMSG_ADDR_ANY; 1596 chinfo.dst = RPMSG_ADDR_ANY; 1597 1598 rpmsg_unregister_device(glink->dev, &chinfo); 1599 } 1600 channel->rpdev = NULL; 1601 1602 kref_put(&channel->refcount, qcom_glink_channel_release); 1603 } 1604 1605 static void qcom_glink_work(struct work_struct *work) 1606 { 1607 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1608 rx_work); 1609 struct glink_defer_cmd *dcmd; 1610 struct glink_msg *msg; 1611 unsigned long flags; 1612 unsigned int param1; 1613 unsigned int param2; 1614 unsigned int cmd; 1615 1616 for (;;) { 1617 spin_lock_irqsave(&glink->rx_lock, flags); 1618 if (list_empty(&glink->rx_queue)) { 1619 spin_unlock_irqrestore(&glink->rx_lock, flags); 1620 break; 1621 } 1622 dcmd = list_first_entry(&glink->rx_queue, 1623 struct glink_defer_cmd, node); 1624 list_del(&dcmd->node); 1625 spin_unlock_irqrestore(&glink->rx_lock, flags); 1626 1627 msg = &dcmd->msg; 1628 cmd = le16_to_cpu(msg->cmd); 1629 param1 = le16_to_cpu(msg->param1); 1630 param2 = le32_to_cpu(msg->param2); 1631 1632 switch (cmd) { 1633 case GLINK_CMD_VERSION: 1634 qcom_glink_receive_version(glink, param1, param2); 1635 break; 1636 case GLINK_CMD_VERSION_ACK: 1637 qcom_glink_receive_version_ack(glink, param1, param2); 1638 break; 1639 case GLINK_CMD_OPEN: 1640 qcom_glink_rx_open(glink, param1, msg->data); 1641 break; 1642 case GLINK_CMD_CLOSE: 1643 qcom_glink_rx_close(glink, param1); 1644 break; 1645 case GLINK_CMD_CLOSE_ACK: 1646 qcom_glink_rx_close_ack(glink, param1); 1647 break; 1648 case GLINK_CMD_RX_INTENT_REQ: 1649 qcom_glink_handle_intent_req(glink, param1, param2); 1650 break; 1651 default: 1652 WARN(1, "Unknown defer object %d\n", cmd); 1653 break; 1654 } 1655 1656 kfree(dcmd); 1657 } 1658 } 1659 1660 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1661 { 1662 struct glink_defer_cmd *dcmd; 1663 struct glink_defer_cmd *tmp; 1664 1665 /* cancel any pending deferred rx_work */ 1666 cancel_work_sync(&glink->rx_work); 1667 1668 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1669 kfree(dcmd); 1670 } 1671 1672 static ssize_t rpmsg_name_show(struct device *dev, 1673 struct device_attribute *attr, char *buf) 1674 { 1675 int ret = 0; 1676 const char *name; 1677 1678 ret = of_property_read_string(dev->of_node, "label", &name); 1679 if (ret < 0) 1680 name = dev->of_node->name; 1681 1682 return sysfs_emit(buf, "%s\n", name); 1683 } 1684 static DEVICE_ATTR_RO(rpmsg_name); 1685 1686 static struct attribute *qcom_glink_attrs[] = { 1687 &dev_attr_rpmsg_name.attr, 1688 NULL 1689 }; 1690 ATTRIBUTE_GROUPS(qcom_glink); 1691 1692 static void qcom_glink_device_release(struct device *dev) 1693 { 1694 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1695 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1696 1697 /* Release qcom_glink_alloc_channel() reference */ 1698 kref_put(&channel->refcount, qcom_glink_channel_release); 1699 kfree(rpdev->driver_override); 1700 kfree(rpdev); 1701 } 1702 1703 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1704 { 1705 struct rpmsg_device *rpdev; 1706 struct glink_channel *channel; 1707 1708 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1709 if (!rpdev) 1710 return -ENOMEM; 1711 1712 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1713 if (IS_ERR(channel)) { 1714 kfree(rpdev); 1715 return PTR_ERR(channel); 1716 } 1717 channel->rpdev = rpdev; 1718 1719 rpdev->ept = &channel->ept; 1720 rpdev->ops = &glink_device_ops; 1721 rpdev->dev.parent = glink->dev; 1722 rpdev->dev.release = qcom_glink_device_release; 1723 1724 return rpmsg_ctrldev_register_device(rpdev); 1725 } 1726 1727 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1728 unsigned long features, 1729 struct qcom_glink_pipe *rx, 1730 struct qcom_glink_pipe *tx, 1731 bool intentless) 1732 { 1733 int ret; 1734 struct qcom_glink *glink; 1735 1736 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1737 if (!glink) 1738 return ERR_PTR(-ENOMEM); 1739 1740 glink->dev = dev; 1741 glink->tx_pipe = tx; 1742 glink->rx_pipe = rx; 1743 1744 glink->features = features; 1745 glink->intentless = intentless; 1746 1747 spin_lock_init(&glink->tx_lock); 1748 spin_lock_init(&glink->rx_lock); 1749 INIT_LIST_HEAD(&glink->rx_queue); 1750 INIT_WORK(&glink->rx_work, qcom_glink_work); 1751 init_waitqueue_head(&glink->tx_avail_notify); 1752 1753 spin_lock_init(&glink->idr_lock); 1754 idr_init(&glink->lcids); 1755 idr_init(&glink->rcids); 1756 1757 glink->dev->groups = qcom_glink_groups; 1758 1759 ret = device_add_groups(dev, qcom_glink_groups); 1760 if (ret) 1761 dev_err(dev, "failed to add groups\n"); 1762 1763 ret = qcom_glink_send_version(glink); 1764 if (ret) 1765 return ERR_PTR(ret); 1766 1767 ret = qcom_glink_create_chrdev(glink); 1768 if (ret) 1769 dev_err(glink->dev, "failed to register chrdev\n"); 1770 1771 return glink; 1772 } 1773 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1774 1775 static int qcom_glink_remove_device(struct device *dev, void *data) 1776 { 1777 device_unregister(dev); 1778 1779 return 0; 1780 } 1781 1782 void qcom_glink_native_remove(struct qcom_glink *glink) 1783 { 1784 struct glink_channel *channel; 1785 unsigned long flags; 1786 int cid; 1787 int ret; 1788 1789 qcom_glink_cancel_rx_work(glink); 1790 1791 /* Fail all attempts at sending messages */ 1792 spin_lock_irqsave(&glink->tx_lock, flags); 1793 glink->abort_tx = true; 1794 wake_up_all(&glink->tx_avail_notify); 1795 spin_unlock_irqrestore(&glink->tx_lock, flags); 1796 1797 /* Abort any senders waiting for intent requests */ 1798 spin_lock_irqsave(&glink->idr_lock, flags); 1799 idr_for_each_entry(&glink->lcids, channel, cid) 1800 qcom_glink_intent_req_abort(channel); 1801 spin_unlock_irqrestore(&glink->idr_lock, flags); 1802 1803 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1804 if (ret) 1805 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1806 1807 /* Release any defunct local channels, waiting for close-ack */ 1808 idr_for_each_entry(&glink->lcids, channel, cid) 1809 kref_put(&channel->refcount, qcom_glink_channel_release); 1810 1811 /* Release any defunct local channels, waiting for close-req */ 1812 idr_for_each_entry(&glink->rcids, channel, cid) 1813 kref_put(&channel->refcount, qcom_glink_channel_release); 1814 1815 idr_destroy(&glink->lcids); 1816 idr_destroy(&glink->rcids); 1817 } 1818 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1819 1820 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1821 MODULE_LICENSE("GPL v2"); 1822