1 /* 2 * ISHTP client logic 3 * 4 * Copyright (c) 2003-2016, Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 */ 16 17 #include <linux/slab.h> 18 #include <linux/sched.h> 19 #include <linux/wait.h> 20 #include <linux/delay.h> 21 #include <linux/dma-mapping.h> 22 #include "hbm.h" 23 #include "client.h" 24 25 /** 26 * ishtp_read_list_flush() - Flush read queue 27 * @cl: ishtp client instance 28 * 29 * Used to remove all entries from read queue for a client 30 */ 31 static void ishtp_read_list_flush(struct ishtp_cl *cl) 32 { 33 struct ishtp_cl_rb *rb; 34 struct ishtp_cl_rb *next; 35 unsigned long flags; 36 37 spin_lock_irqsave(&cl->dev->read_list_spinlock, flags); 38 list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list) 39 if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) { 40 list_del(&rb->list); 41 ishtp_io_rb_free(rb); 42 } 43 spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags); 44 } 45 46 /** 47 * ishtp_cl_flush_queues() - Flush all queues for a client 48 * @cl: ishtp client instance 49 * 50 * Used to remove all queues for a client. This is called when a client device 51 * needs reset due to error, S3 resume or during module removal 52 * 53 * Return: 0 on success else -EINVAL if device is NULL 54 */ 55 int ishtp_cl_flush_queues(struct ishtp_cl *cl) 56 { 57 if (WARN_ON(!cl || !cl->dev)) 58 return -EINVAL; 59 60 ishtp_read_list_flush(cl); 61 62 return 0; 63 } 64 EXPORT_SYMBOL(ishtp_cl_flush_queues); 65 66 /** 67 * ishtp_cl_init() - Initialize all fields of a client device 68 * @cl: ishtp client instance 69 * @dev: ishtp device 70 * 71 * Initializes a client device fields: Init spinlocks, init queues etc. 72 * This function is called during new client creation 73 */ 74 static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev) 75 { 76 memset(cl, 0, sizeof(struct ishtp_cl)); 77 init_waitqueue_head(&cl->wait_ctrl_res); 78 spin_lock_init(&cl->free_list_spinlock); 79 spin_lock_init(&cl->in_process_spinlock); 80 spin_lock_init(&cl->tx_list_spinlock); 81 spin_lock_init(&cl->tx_free_list_spinlock); 82 spin_lock_init(&cl->fc_spinlock); 83 INIT_LIST_HEAD(&cl->link); 84 cl->dev = dev; 85 86 INIT_LIST_HEAD(&cl->free_rb_list.list); 87 INIT_LIST_HEAD(&cl->tx_list.list); 88 INIT_LIST_HEAD(&cl->tx_free_list.list); 89 INIT_LIST_HEAD(&cl->in_process_list.list); 90 91 cl->rx_ring_size = CL_DEF_RX_RING_SIZE; 92 cl->tx_ring_size = CL_DEF_TX_RING_SIZE; 93 94 /* dma */ 95 cl->last_tx_path = CL_TX_PATH_IPC; 96 cl->last_dma_acked = 1; 97 cl->last_dma_addr = NULL; 98 cl->last_ipc_acked = 1; 99 } 100 101 /** 102 * ishtp_cl_allocate() - allocates client structure and sets it up. 103 * @dev: ishtp device 104 * 105 * Allocate memory for new client device and call to initialize each field. 106 * 107 * Return: The allocated client instance or NULL on failure 108 */ 109 struct ishtp_cl *ishtp_cl_allocate(struct ishtp_device *dev) 110 { 111 struct ishtp_cl *cl; 112 113 cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL); 114 if (!cl) 115 return NULL; 116 117 ishtp_cl_init(cl, dev); 118 return cl; 119 } 120 EXPORT_SYMBOL(ishtp_cl_allocate); 121 122 /** 123 * ishtp_cl_free() - Frees a client device 124 * @cl: client device instance 125 * 126 * Frees a client device 127 */ 128 void ishtp_cl_free(struct ishtp_cl *cl) 129 { 130 struct ishtp_device *dev; 131 unsigned long flags; 132 133 if (!cl) 134 return; 135 136 dev = cl->dev; 137 if (!dev) 138 return; 139 140 spin_lock_irqsave(&dev->cl_list_lock, flags); 141 ishtp_cl_free_rx_ring(cl); 142 ishtp_cl_free_tx_ring(cl); 143 kfree(cl); 144 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 145 } 146 EXPORT_SYMBOL(ishtp_cl_free); 147 148 /** 149 * ishtp_cl_link() - Reserve a host id and link the client instance 150 * @cl: client device instance 151 * @id: host client id to use. It can be ISHTP_HOST_CLIENT_ID_ANY if any 152 * id from the available can be used 153 * 154 * 155 * This allocates a single bit in the hostmap. This function will make sure 156 * that not many client sessions are opened at the same time. Once allocated 157 * the client device instance is added to the ishtp device in the current 158 * client list 159 * 160 * Return: 0 or error code on failure 161 */ 162 int ishtp_cl_link(struct ishtp_cl *cl, int id) 163 { 164 struct ishtp_device *dev; 165 unsigned long flags, flags_cl; 166 int ret = 0; 167 168 if (WARN_ON(!cl || !cl->dev)) 169 return -EINVAL; 170 171 dev = cl->dev; 172 173 spin_lock_irqsave(&dev->device_lock, flags); 174 175 if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) { 176 ret = -EMFILE; 177 goto unlock_dev; 178 } 179 180 /* If Id is not assigned get one*/ 181 if (id == ISHTP_HOST_CLIENT_ID_ANY) 182 id = find_first_zero_bit(dev->host_clients_map, 183 ISHTP_CLIENTS_MAX); 184 185 if (id >= ISHTP_CLIENTS_MAX) { 186 spin_unlock_irqrestore(&dev->device_lock, flags); 187 dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX); 188 return -ENOENT; 189 } 190 191 dev->open_handle_count++; 192 cl->host_client_id = id; 193 spin_lock_irqsave(&dev->cl_list_lock, flags_cl); 194 if (dev->dev_state != ISHTP_DEV_ENABLED) { 195 ret = -ENODEV; 196 goto unlock_cl; 197 } 198 list_add_tail(&cl->link, &dev->cl_list); 199 set_bit(id, dev->host_clients_map); 200 cl->state = ISHTP_CL_INITIALIZING; 201 202 unlock_cl: 203 spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl); 204 unlock_dev: 205 spin_unlock_irqrestore(&dev->device_lock, flags); 206 return ret; 207 } 208 EXPORT_SYMBOL(ishtp_cl_link); 209 210 /** 211 * ishtp_cl_unlink() - remove fw_cl from the client device list 212 * @cl: client device instance 213 * 214 * Remove a previously linked device to a ishtp device 215 */ 216 void ishtp_cl_unlink(struct ishtp_cl *cl) 217 { 218 struct ishtp_device *dev; 219 struct ishtp_cl *pos; 220 unsigned long flags; 221 222 /* don't shout on error exit path */ 223 if (!cl || !cl->dev) 224 return; 225 226 dev = cl->dev; 227 228 spin_lock_irqsave(&dev->device_lock, flags); 229 if (dev->open_handle_count > 0) { 230 clear_bit(cl->host_client_id, dev->host_clients_map); 231 dev->open_handle_count--; 232 } 233 spin_unlock_irqrestore(&dev->device_lock, flags); 234 235 /* 236 * This checks that 'cl' is actually linked into device's structure, 237 * before attempting 'list_del' 238 */ 239 spin_lock_irqsave(&dev->cl_list_lock, flags); 240 list_for_each_entry(pos, &dev->cl_list, link) 241 if (cl->host_client_id == pos->host_client_id) { 242 list_del_init(&pos->link); 243 break; 244 } 245 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 246 } 247 EXPORT_SYMBOL(ishtp_cl_unlink); 248 249 /** 250 * ishtp_cl_disconnect() - Send disconnect request to firmware 251 * @cl: client device instance 252 * 253 * Send a disconnect request for a client to firmware. 254 * 255 * Return: 0 if successful disconnect response from the firmware or error 256 * code on failure 257 */ 258 int ishtp_cl_disconnect(struct ishtp_cl *cl) 259 { 260 struct ishtp_device *dev; 261 int err; 262 263 if (WARN_ON(!cl || !cl->dev)) 264 return -ENODEV; 265 266 dev = cl->dev; 267 268 dev->print_log(dev, "%s() state %d\n", __func__, cl->state); 269 270 if (cl->state != ISHTP_CL_DISCONNECTING) { 271 dev->print_log(dev, "%s() Disconnect in progress\n", __func__); 272 return 0; 273 } 274 275 if (ishtp_hbm_cl_disconnect_req(dev, cl)) { 276 dev->print_log(dev, "%s() Failed to disconnect\n", __func__); 277 dev_err(&cl->device->dev, "failed to disconnect.\n"); 278 return -ENODEV; 279 } 280 281 err = wait_event_interruptible_timeout(cl->wait_ctrl_res, 282 (dev->dev_state != ISHTP_DEV_ENABLED || 283 cl->state == ISHTP_CL_DISCONNECTED), 284 ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT)); 285 286 /* 287 * If FW reset arrived, this will happen. Don't check cl->, 288 * as 'cl' may be freed already 289 */ 290 if (dev->dev_state != ISHTP_DEV_ENABLED) { 291 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n", 292 __func__); 293 return -ENODEV; 294 } 295 296 if (cl->state == ISHTP_CL_DISCONNECTED) { 297 dev->print_log(dev, "%s() successful\n", __func__); 298 return 0; 299 } 300 301 return -ENODEV; 302 } 303 EXPORT_SYMBOL(ishtp_cl_disconnect); 304 305 /** 306 * ishtp_cl_is_other_connecting() - Check other client is connecting 307 * @cl: client device instance 308 * 309 * Checks if other client with the same fw client id is connecting 310 * 311 * Return: true if other client is connected else false 312 */ 313 static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl) 314 { 315 struct ishtp_device *dev; 316 struct ishtp_cl *pos; 317 unsigned long flags; 318 319 if (WARN_ON(!cl || !cl->dev)) 320 return false; 321 322 dev = cl->dev; 323 spin_lock_irqsave(&dev->cl_list_lock, flags); 324 list_for_each_entry(pos, &dev->cl_list, link) { 325 if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) && 326 cl->fw_client_id == pos->fw_client_id) { 327 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 328 return true; 329 } 330 } 331 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 332 333 return false; 334 } 335 336 /** 337 * ishtp_cl_connect() - Send connect request to firmware 338 * @cl: client device instance 339 * 340 * Send a connect request for a client to firmware. If successful it will 341 * RX and TX ring buffers 342 * 343 * Return: 0 if successful connect response from the firmware and able 344 * to bind and allocate ring buffers or error code on failure 345 */ 346 int ishtp_cl_connect(struct ishtp_cl *cl) 347 { 348 struct ishtp_device *dev; 349 int rets; 350 351 if (WARN_ON(!cl || !cl->dev)) 352 return -ENODEV; 353 354 dev = cl->dev; 355 356 dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state); 357 358 if (ishtp_cl_is_other_connecting(cl)) { 359 dev->print_log(dev, "%s() Busy\n", __func__); 360 return -EBUSY; 361 } 362 363 if (ishtp_hbm_cl_connect_req(dev, cl)) { 364 dev->print_log(dev, "%s() HBM connect req fail\n", __func__); 365 return -ENODEV; 366 } 367 368 rets = wait_event_interruptible_timeout(cl->wait_ctrl_res, 369 (dev->dev_state == ISHTP_DEV_ENABLED && 370 (cl->state == ISHTP_CL_CONNECTED || 371 cl->state == ISHTP_CL_DISCONNECTED)), 372 ishtp_secs_to_jiffies( 373 ISHTP_CL_CONNECT_TIMEOUT)); 374 /* 375 * If FW reset arrived, this will happen. Don't check cl->, 376 * as 'cl' may be freed already 377 */ 378 if (dev->dev_state != ISHTP_DEV_ENABLED) { 379 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n", 380 __func__); 381 return -EFAULT; 382 } 383 384 if (cl->state != ISHTP_CL_CONNECTED) { 385 dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n", 386 __func__); 387 return -EFAULT; 388 } 389 390 rets = cl->status; 391 if (rets) { 392 dev->print_log(dev, "%s() Invalid status\n", __func__); 393 return rets; 394 } 395 396 rets = ishtp_cl_device_bind(cl); 397 if (rets) { 398 dev->print_log(dev, "%s() Bind error\n", __func__); 399 ishtp_cl_disconnect(cl); 400 return rets; 401 } 402 403 rets = ishtp_cl_alloc_rx_ring(cl); 404 if (rets) { 405 dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__); 406 /* if failed allocation, disconnect */ 407 ishtp_cl_disconnect(cl); 408 return rets; 409 } 410 411 rets = ishtp_cl_alloc_tx_ring(cl); 412 if (rets) { 413 dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__); 414 /* if failed allocation, disconnect */ 415 ishtp_cl_free_rx_ring(cl); 416 ishtp_cl_disconnect(cl); 417 return rets; 418 } 419 420 /* Upon successful connection and allocation, emit flow-control */ 421 rets = ishtp_cl_read_start(cl); 422 423 dev->print_log(dev, "%s() successful\n", __func__); 424 425 return rets; 426 } 427 EXPORT_SYMBOL(ishtp_cl_connect); 428 429 /** 430 * ishtp_cl_read_start() - Prepare to read client message 431 * @cl: client device instance 432 * 433 * Get a free buffer from pool of free read buffers and add to read buffer 434 * pool to add contents. Send a flow control request to firmware to be able 435 * send next message. 436 * 437 * Return: 0 if successful or error code on failure 438 */ 439 int ishtp_cl_read_start(struct ishtp_cl *cl) 440 { 441 struct ishtp_device *dev; 442 struct ishtp_cl_rb *rb; 443 int rets; 444 int i; 445 unsigned long flags; 446 unsigned long dev_flags; 447 448 if (WARN_ON(!cl || !cl->dev)) 449 return -ENODEV; 450 451 dev = cl->dev; 452 453 if (cl->state != ISHTP_CL_CONNECTED) 454 return -ENODEV; 455 456 if (dev->dev_state != ISHTP_DEV_ENABLED) 457 return -ENODEV; 458 459 i = ishtp_fw_cl_by_id(dev, cl->fw_client_id); 460 if (i < 0) { 461 dev_err(&cl->device->dev, "no such fw client %d\n", 462 cl->fw_client_id); 463 return -ENODEV; 464 } 465 466 /* The current rb is the head of the free rb list */ 467 spin_lock_irqsave(&cl->free_list_spinlock, flags); 468 if (list_empty(&cl->free_rb_list.list)) { 469 dev_warn(&cl->device->dev, 470 "[ishtp-ish] Rx buffers pool is empty\n"); 471 rets = -ENOMEM; 472 rb = NULL; 473 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 474 goto out; 475 } 476 rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list); 477 list_del_init(&rb->list); 478 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 479 480 rb->cl = cl; 481 rb->buf_idx = 0; 482 483 INIT_LIST_HEAD(&rb->list); 484 rets = 0; 485 486 /* 487 * This must be BEFORE sending flow control - 488 * response in ISR may come too fast... 489 */ 490 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags); 491 list_add_tail(&rb->list, &dev->read_list.list); 492 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags); 493 if (ishtp_hbm_cl_flow_control_req(dev, cl)) { 494 rets = -ENODEV; 495 goto out; 496 } 497 out: 498 /* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */ 499 if (rets && rb) { 500 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags); 501 list_del(&rb->list); 502 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags); 503 504 spin_lock_irqsave(&cl->free_list_spinlock, flags); 505 list_add_tail(&rb->list, &cl->free_rb_list.list); 506 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 507 } 508 return rets; 509 } 510 511 /** 512 * ishtp_cl_send() - Send a message to firmware 513 * @cl: client device instance 514 * @buf: message buffer 515 * @length: length of message 516 * 517 * If the client is correct state to send message, this function gets a buffer 518 * from tx ring buffers, copy the message data and call to send the message 519 * using ishtp_cl_send_msg() 520 * 521 * Return: 0 if successful or error code on failure 522 */ 523 int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length) 524 { 525 struct ishtp_device *dev; 526 int id; 527 struct ishtp_cl_tx_ring *cl_msg; 528 int have_msg_to_send = 0; 529 unsigned long tx_flags, tx_free_flags; 530 531 if (WARN_ON(!cl || !cl->dev)) 532 return -ENODEV; 533 534 dev = cl->dev; 535 536 if (cl->state != ISHTP_CL_CONNECTED) { 537 ++cl->err_send_msg; 538 return -EPIPE; 539 } 540 541 if (dev->dev_state != ISHTP_DEV_ENABLED) { 542 ++cl->err_send_msg; 543 return -ENODEV; 544 } 545 546 /* Check if we have fw client device */ 547 id = ishtp_fw_cl_by_id(dev, cl->fw_client_id); 548 if (id < 0) { 549 ++cl->err_send_msg; 550 return -ENOENT; 551 } 552 553 if (length > dev->fw_clients[id].props.max_msg_length) { 554 ++cl->err_send_msg; 555 return -EMSGSIZE; 556 } 557 558 /* No free bufs */ 559 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 560 if (list_empty(&cl->tx_free_list.list)) { 561 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 562 tx_free_flags); 563 ++cl->err_send_msg; 564 return -ENOMEM; 565 } 566 567 cl_msg = list_first_entry(&cl->tx_free_list.list, 568 struct ishtp_cl_tx_ring, list); 569 if (!cl_msg->send_buf.data) { 570 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 571 tx_free_flags); 572 return -EIO; 573 /* Should not happen, as free list is pre-allocated */ 574 } 575 /* 576 * This is safe, as 'length' is already checked for not exceeding 577 * max ISHTP message size per client 578 */ 579 list_del_init(&cl_msg->list); 580 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags); 581 memcpy(cl_msg->send_buf.data, buf, length); 582 cl_msg->send_buf.size = length; 583 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 584 have_msg_to_send = !list_empty(&cl->tx_list.list); 585 list_add_tail(&cl_msg->list, &cl->tx_list.list); 586 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 587 588 if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0) 589 ishtp_cl_send_msg(dev, cl); 590 591 return 0; 592 } 593 EXPORT_SYMBOL(ishtp_cl_send); 594 595 /** 596 * ishtp_cl_read_complete() - read complete 597 * @rb: Pointer to client request block 598 * 599 * If the message is completely received call ishtp_cl_bus_rx_event() 600 * to process message 601 */ 602 static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb) 603 { 604 unsigned long flags; 605 int schedule_work_flag = 0; 606 struct ishtp_cl *cl = rb->cl; 607 608 spin_lock_irqsave(&cl->in_process_spinlock, flags); 609 /* 610 * if in-process list is empty, then need to schedule 611 * the processing thread 612 */ 613 schedule_work_flag = list_empty(&cl->in_process_list.list); 614 list_add_tail(&rb->list, &cl->in_process_list.list); 615 spin_unlock_irqrestore(&cl->in_process_spinlock, flags); 616 617 if (schedule_work_flag) 618 ishtp_cl_bus_rx_event(cl->device); 619 } 620 621 /** 622 * ipc_tx_callback() - IPC tx callback function 623 * @prm: Pointer to client device instance 624 * 625 * Send message over IPC either first time or on callback on previous message 626 * completion 627 */ 628 static void ipc_tx_callback(void *prm) 629 { 630 struct ishtp_cl *cl = prm; 631 struct ishtp_cl_tx_ring *cl_msg; 632 size_t rem; 633 struct ishtp_device *dev = (cl ? cl->dev : NULL); 634 struct ishtp_msg_hdr ishtp_hdr; 635 unsigned long tx_flags, tx_free_flags; 636 unsigned char *pmsg; 637 638 if (!dev) 639 return; 640 641 /* 642 * Other conditions if some critical error has 643 * occurred before this callback is called 644 */ 645 if (dev->dev_state != ISHTP_DEV_ENABLED) 646 return; 647 648 if (cl->state != ISHTP_CL_CONNECTED) 649 return; 650 651 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 652 if (list_empty(&cl->tx_list.list)) { 653 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 654 return; 655 } 656 657 if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) { 658 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 659 return; 660 } 661 662 if (!cl->sending) { 663 --cl->ishtp_flow_ctrl_creds; 664 cl->last_ipc_acked = 0; 665 cl->last_tx_path = CL_TX_PATH_IPC; 666 cl->sending = 1; 667 } 668 669 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring, 670 list); 671 rem = cl_msg->send_buf.size - cl->tx_offs; 672 673 ishtp_hdr.host_addr = cl->host_client_id; 674 ishtp_hdr.fw_addr = cl->fw_client_id; 675 ishtp_hdr.reserved = 0; 676 pmsg = cl_msg->send_buf.data + cl->tx_offs; 677 678 if (rem <= dev->mtu) { 679 ishtp_hdr.length = rem; 680 ishtp_hdr.msg_complete = 1; 681 cl->sending = 0; 682 list_del_init(&cl_msg->list); /* Must be before write */ 683 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 684 /* Submit to IPC queue with no callback */ 685 ishtp_write_message(dev, &ishtp_hdr, pmsg); 686 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 687 list_add_tail(&cl_msg->list, &cl->tx_free_list.list); 688 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 689 tx_free_flags); 690 } else { 691 /* Send IPC fragment */ 692 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 693 cl->tx_offs += dev->mtu; 694 ishtp_hdr.length = dev->mtu; 695 ishtp_hdr.msg_complete = 0; 696 ishtp_send_msg(dev, &ishtp_hdr, pmsg, ipc_tx_callback, cl); 697 } 698 } 699 700 /** 701 * ishtp_cl_send_msg_ipc() -Send message using IPC 702 * @dev: ISHTP device instance 703 * @cl: Pointer to client device instance 704 * 705 * Send message over IPC not using DMA 706 */ 707 static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev, 708 struct ishtp_cl *cl) 709 { 710 /* If last DMA message wasn't acked yet, leave this one in Tx queue */ 711 if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0) 712 return; 713 714 cl->tx_offs = 0; 715 ipc_tx_callback(cl); 716 ++cl->send_msg_cnt_ipc; 717 } 718 719 /** 720 * ishtp_cl_send_msg_dma() -Send message using DMA 721 * @dev: ISHTP device instance 722 * @cl: Pointer to client device instance 723 * 724 * Send message using DMA 725 */ 726 static void ishtp_cl_send_msg_dma(struct ishtp_device *dev, 727 struct ishtp_cl *cl) 728 { 729 struct ishtp_msg_hdr hdr; 730 struct dma_xfer_hbm dma_xfer; 731 unsigned char *msg_addr; 732 int off; 733 struct ishtp_cl_tx_ring *cl_msg; 734 unsigned long tx_flags, tx_free_flags; 735 736 /* If last IPC message wasn't acked yet, leave this one in Tx queue */ 737 if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0) 738 return; 739 740 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 741 if (list_empty(&cl->tx_list.list)) { 742 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 743 return; 744 } 745 746 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring, 747 list); 748 749 msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size); 750 if (!msg_addr) { 751 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 752 if (dev->transfer_path == CL_TX_PATH_DEFAULT) 753 ishtp_cl_send_msg_ipc(dev, cl); 754 return; 755 } 756 757 list_del_init(&cl_msg->list); /* Must be before write */ 758 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 759 760 --cl->ishtp_flow_ctrl_creds; 761 cl->last_dma_acked = 0; 762 cl->last_dma_addr = msg_addr; 763 cl->last_tx_path = CL_TX_PATH_DMA; 764 765 /* write msg to dma buf */ 766 memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size); 767 768 /* send dma_xfer hbm msg */ 769 off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf; 770 ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm)); 771 dma_xfer.hbm = DMA_XFER; 772 dma_xfer.fw_client_id = cl->fw_client_id; 773 dma_xfer.host_client_id = cl->host_client_id; 774 dma_xfer.reserved = 0; 775 dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off; 776 dma_xfer.msg_length = cl_msg->send_buf.size; 777 dma_xfer.reserved2 = 0; 778 ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer); 779 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 780 list_add_tail(&cl_msg->list, &cl->tx_free_list.list); 781 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags); 782 ++cl->send_msg_cnt_dma; 783 } 784 785 /** 786 * ishtp_cl_send_msg() -Send message using DMA or IPC 787 * @dev: ISHTP device instance 788 * @cl: Pointer to client device instance 789 * 790 * Send message using DMA or IPC based on transfer_path 791 */ 792 void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl) 793 { 794 if (dev->transfer_path == CL_TX_PATH_DMA) 795 ishtp_cl_send_msg_dma(dev, cl); 796 else 797 ishtp_cl_send_msg_ipc(dev, cl); 798 } 799 800 /** 801 * recv_ishtp_cl_msg() -Receive client message 802 * @dev: ISHTP device instance 803 * @ishtp_hdr: Pointer to message header 804 * 805 * Receive and dispatch ISHTP client messages. This function executes in ISR 806 * or work queue context 807 */ 808 void recv_ishtp_cl_msg(struct ishtp_device *dev, 809 struct ishtp_msg_hdr *ishtp_hdr) 810 { 811 struct ishtp_cl *cl; 812 struct ishtp_cl_rb *rb; 813 struct ishtp_cl_rb *new_rb; 814 unsigned char *buffer = NULL; 815 struct ishtp_cl_rb *complete_rb = NULL; 816 unsigned long flags; 817 int rb_count; 818 819 if (ishtp_hdr->reserved) { 820 dev_err(dev->devc, "corrupted message header.\n"); 821 goto eoi; 822 } 823 824 if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) { 825 dev_err(dev->devc, 826 "ISHTP message length in hdr exceeds IPC MTU\n"); 827 goto eoi; 828 } 829 830 spin_lock_irqsave(&dev->read_list_spinlock, flags); 831 rb_count = -1; 832 list_for_each_entry(rb, &dev->read_list.list, list) { 833 ++rb_count; 834 cl = rb->cl; 835 if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr && 836 cl->fw_client_id == ishtp_hdr->fw_addr) || 837 !(cl->state == ISHTP_CL_CONNECTED)) 838 continue; 839 840 /* If no Rx buffer is allocated, disband the rb */ 841 if (rb->buffer.size == 0 || rb->buffer.data == NULL) { 842 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 843 dev_err(&cl->device->dev, 844 "Rx buffer is not allocated.\n"); 845 list_del(&rb->list); 846 ishtp_io_rb_free(rb); 847 cl->status = -ENOMEM; 848 goto eoi; 849 } 850 851 /* 852 * If message buffer overflown (exceeds max. client msg 853 * size, drop message and return to free buffer. 854 * Do we need to disconnect such a client? (We don't send 855 * back FC, so communication will be stuck anyway) 856 */ 857 if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) { 858 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 859 dev_err(&cl->device->dev, 860 "message overflow. size %d len %d idx %ld\n", 861 rb->buffer.size, ishtp_hdr->length, 862 rb->buf_idx); 863 list_del(&rb->list); 864 ishtp_cl_io_rb_recycle(rb); 865 cl->status = -EIO; 866 goto eoi; 867 } 868 869 buffer = rb->buffer.data + rb->buf_idx; 870 dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length); 871 872 rb->buf_idx += ishtp_hdr->length; 873 if (ishtp_hdr->msg_complete) { 874 /* Last fragment in message - it's complete */ 875 cl->status = 0; 876 list_del(&rb->list); 877 complete_rb = rb; 878 879 --cl->out_flow_ctrl_creds; 880 /* 881 * the whole msg arrived, send a new FC, and add a new 882 * rb buffer for the next coming msg 883 */ 884 spin_lock(&cl->free_list_spinlock); 885 886 if (!list_empty(&cl->free_rb_list.list)) { 887 new_rb = list_entry(cl->free_rb_list.list.next, 888 struct ishtp_cl_rb, list); 889 list_del_init(&new_rb->list); 890 spin_unlock(&cl->free_list_spinlock); 891 new_rb->cl = cl; 892 new_rb->buf_idx = 0; 893 INIT_LIST_HEAD(&new_rb->list); 894 list_add_tail(&new_rb->list, 895 &dev->read_list.list); 896 897 ishtp_hbm_cl_flow_control_req(dev, cl); 898 } else { 899 spin_unlock(&cl->free_list_spinlock); 900 } 901 } 902 /* One more fragment in message (even if this was last) */ 903 ++cl->recv_msg_num_frags; 904 905 /* 906 * We can safely break here (and in BH too), 907 * a single input message can go only to a single request! 908 */ 909 break; 910 } 911 912 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 913 /* If it's nobody's message, just read and discard it */ 914 if (!buffer) { 915 uint8_t rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE]; 916 917 dev_err(dev->devc, "Dropped Rx msg - no request\n"); 918 dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length); 919 goto eoi; 920 } 921 922 if (complete_rb) { 923 cl = complete_rb->cl; 924 cl->ts_rx = ktime_get(); 925 ++cl->recv_msg_cnt_ipc; 926 ishtp_cl_read_complete(complete_rb); 927 } 928 eoi: 929 return; 930 } 931 932 /** 933 * recv_ishtp_cl_msg_dma() -Receive client message 934 * @dev: ISHTP device instance 935 * @msg: message pointer 936 * @hbm: hbm buffer 937 * 938 * Receive and dispatch ISHTP client messages using DMA. This function executes 939 * in ISR or work queue context 940 */ 941 void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg, 942 struct dma_xfer_hbm *hbm) 943 { 944 struct ishtp_cl *cl; 945 struct ishtp_cl_rb *rb; 946 struct ishtp_cl_rb *new_rb; 947 unsigned char *buffer = NULL; 948 struct ishtp_cl_rb *complete_rb = NULL; 949 unsigned long flags; 950 951 spin_lock_irqsave(&dev->read_list_spinlock, flags); 952 953 list_for_each_entry(rb, &dev->read_list.list, list) { 954 cl = rb->cl; 955 if (!cl || !(cl->host_client_id == hbm->host_client_id && 956 cl->fw_client_id == hbm->fw_client_id) || 957 !(cl->state == ISHTP_CL_CONNECTED)) 958 continue; 959 960 /* 961 * If no Rx buffer is allocated, disband the rb 962 */ 963 if (rb->buffer.size == 0 || rb->buffer.data == NULL) { 964 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 965 dev_err(&cl->device->dev, 966 "response buffer is not allocated.\n"); 967 list_del(&rb->list); 968 ishtp_io_rb_free(rb); 969 cl->status = -ENOMEM; 970 goto eoi; 971 } 972 973 /* 974 * If message buffer overflown (exceeds max. client msg 975 * size, drop message and return to free buffer. 976 * Do we need to disconnect such a client? (We don't send 977 * back FC, so communication will be stuck anyway) 978 */ 979 if (rb->buffer.size < hbm->msg_length) { 980 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 981 dev_err(&cl->device->dev, 982 "message overflow. size %d len %d idx %ld\n", 983 rb->buffer.size, hbm->msg_length, rb->buf_idx); 984 list_del(&rb->list); 985 ishtp_cl_io_rb_recycle(rb); 986 cl->status = -EIO; 987 goto eoi; 988 } 989 990 buffer = rb->buffer.data; 991 memcpy(buffer, msg, hbm->msg_length); 992 rb->buf_idx = hbm->msg_length; 993 994 /* Last fragment in message - it's complete */ 995 cl->status = 0; 996 list_del(&rb->list); 997 complete_rb = rb; 998 999 --cl->out_flow_ctrl_creds; 1000 /* 1001 * the whole msg arrived, send a new FC, and add a new 1002 * rb buffer for the next coming msg 1003 */ 1004 spin_lock(&cl->free_list_spinlock); 1005 1006 if (!list_empty(&cl->free_rb_list.list)) { 1007 new_rb = list_entry(cl->free_rb_list.list.next, 1008 struct ishtp_cl_rb, list); 1009 list_del_init(&new_rb->list); 1010 spin_unlock(&cl->free_list_spinlock); 1011 new_rb->cl = cl; 1012 new_rb->buf_idx = 0; 1013 INIT_LIST_HEAD(&new_rb->list); 1014 list_add_tail(&new_rb->list, 1015 &dev->read_list.list); 1016 1017 ishtp_hbm_cl_flow_control_req(dev, cl); 1018 } else { 1019 spin_unlock(&cl->free_list_spinlock); 1020 } 1021 1022 /* One more fragment in message (this is always last) */ 1023 ++cl->recv_msg_num_frags; 1024 1025 /* 1026 * We can safely break here (and in BH too), 1027 * a single input message can go only to a single request! 1028 */ 1029 break; 1030 } 1031 1032 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 1033 /* If it's nobody's message, just read and discard it */ 1034 if (!buffer) { 1035 dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n"); 1036 goto eoi; 1037 } 1038 1039 if (complete_rb) { 1040 cl = complete_rb->cl; 1041 cl->ts_rx = ktime_get(); 1042 ++cl->recv_msg_cnt_dma; 1043 ishtp_cl_read_complete(complete_rb); 1044 } 1045 eoi: 1046 return; 1047 } 1048