1 /*
2  * ISHTP client logic
3  *
4  * Copyright (c) 2003-2016, Intel Corporation.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13  * more details.
14  *
15  */
16 
17 #include <linux/slab.h>
18 #include <linux/sched.h>
19 #include <linux/wait.h>
20 #include <linux/delay.h>
21 #include <linux/dma-mapping.h>
22 #include "hbm.h"
23 #include "client.h"
24 
25 /**
26  * ishtp_read_list_flush() - Flush read queue
27  * @cl: ishtp client instance
28  *
29  * Used to remove all entries from read queue for a client
30  */
31 static void ishtp_read_list_flush(struct ishtp_cl *cl)
32 {
33 	struct ishtp_cl_rb *rb;
34 	struct ishtp_cl_rb *next;
35 	unsigned long	flags;
36 
37 	spin_lock_irqsave(&cl->dev->read_list_spinlock, flags);
38 	list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list)
39 		if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) {
40 			list_del(&rb->list);
41 			ishtp_io_rb_free(rb);
42 		}
43 	spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags);
44 }
45 
46 /**
47  * ishtp_cl_flush_queues() - Flush all queues for a client
48  * @cl: ishtp client instance
49  *
50  * Used to remove all queues for a client. This is called when a client device
51  * needs reset due to error, S3 resume or during module removal
52  *
53  * Return: 0 on success else -EINVAL if device is NULL
54  */
55 int ishtp_cl_flush_queues(struct ishtp_cl *cl)
56 {
57 	if (WARN_ON(!cl || !cl->dev))
58 		return -EINVAL;
59 
60 	ishtp_read_list_flush(cl);
61 
62 	return 0;
63 }
64 EXPORT_SYMBOL(ishtp_cl_flush_queues);
65 
66 /**
67  * ishtp_cl_init() - Initialize all fields of a client device
68  * @cl: ishtp client instance
69  * @dev: ishtp device
70  *
71  * Initializes a client device fields: Init spinlocks, init queues etc.
72  * This function is called during new client creation
73  */
74 static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev)
75 {
76 	memset(cl, 0, sizeof(struct ishtp_cl));
77 	init_waitqueue_head(&cl->wait_ctrl_res);
78 	spin_lock_init(&cl->free_list_spinlock);
79 	spin_lock_init(&cl->in_process_spinlock);
80 	spin_lock_init(&cl->tx_list_spinlock);
81 	spin_lock_init(&cl->tx_free_list_spinlock);
82 	spin_lock_init(&cl->fc_spinlock);
83 	INIT_LIST_HEAD(&cl->link);
84 	cl->dev = dev;
85 
86 	INIT_LIST_HEAD(&cl->free_rb_list.list);
87 	INIT_LIST_HEAD(&cl->tx_list.list);
88 	INIT_LIST_HEAD(&cl->tx_free_list.list);
89 	INIT_LIST_HEAD(&cl->in_process_list.list);
90 
91 	cl->rx_ring_size = CL_DEF_RX_RING_SIZE;
92 	cl->tx_ring_size = CL_DEF_TX_RING_SIZE;
93 
94 	/* dma */
95 	cl->last_tx_path = CL_TX_PATH_IPC;
96 	cl->last_dma_acked = 1;
97 	cl->last_dma_addr = NULL;
98 	cl->last_ipc_acked = 1;
99 }
100 
101 /**
102  * ishtp_cl_allocate() - allocates client structure and sets it up.
103  * @dev: ishtp device
104  *
105  * Allocate memory for new client device and call to initialize each field.
106  *
107  * Return: The allocated client instance or NULL on failure
108  */
109 struct ishtp_cl *ishtp_cl_allocate(struct ishtp_device *dev)
110 {
111 	struct ishtp_cl *cl;
112 
113 	cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL);
114 	if (!cl)
115 		return NULL;
116 
117 	ishtp_cl_init(cl, dev);
118 	return cl;
119 }
120 EXPORT_SYMBOL(ishtp_cl_allocate);
121 
122 /**
123  * ishtp_cl_free() - Frees a client device
124  * @cl: client device instance
125  *
126  * Frees a client device
127  */
128 void	ishtp_cl_free(struct ishtp_cl *cl)
129 {
130 	struct ishtp_device *dev;
131 	unsigned long flags;
132 
133 	if (!cl)
134 		return;
135 
136 	dev = cl->dev;
137 	if (!dev)
138 		return;
139 
140 	spin_lock_irqsave(&dev->cl_list_lock, flags);
141 	ishtp_cl_free_rx_ring(cl);
142 	ishtp_cl_free_tx_ring(cl);
143 	kfree(cl);
144 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
145 }
146 EXPORT_SYMBOL(ishtp_cl_free);
147 
148 /**
149  * ishtp_cl_link() - Reserve a host id and link the client instance
150  * @cl: client device instance
151  * @id: host client id to use. It can be ISHTP_HOST_CLIENT_ID_ANY if any
152  *	id from the available can be used
153  *
154  *
155  * This allocates a single bit in the hostmap. This function will make sure
156  * that not many client sessions are opened at the same time. Once allocated
157  * the client device instance is added to the ishtp device in the current
158  * client list
159  *
160  * Return: 0 or error code on failure
161  */
162 int ishtp_cl_link(struct ishtp_cl *cl, int id)
163 {
164 	struct ishtp_device *dev;
165 	unsigned long	flags, flags_cl;
166 	int	ret = 0;
167 
168 	if (WARN_ON(!cl || !cl->dev))
169 		return -EINVAL;
170 
171 	dev = cl->dev;
172 
173 	spin_lock_irqsave(&dev->device_lock, flags);
174 
175 	if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) {
176 		ret = -EMFILE;
177 		goto unlock_dev;
178 	}
179 
180 	/* If Id is not assigned get one*/
181 	if (id == ISHTP_HOST_CLIENT_ID_ANY)
182 		id = find_first_zero_bit(dev->host_clients_map,
183 			ISHTP_CLIENTS_MAX);
184 
185 	if (id >= ISHTP_CLIENTS_MAX) {
186 		spin_unlock_irqrestore(&dev->device_lock, flags);
187 		dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX);
188 		return -ENOENT;
189 	}
190 
191 	dev->open_handle_count++;
192 	cl->host_client_id = id;
193 	spin_lock_irqsave(&dev->cl_list_lock, flags_cl);
194 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
195 		ret = -ENODEV;
196 		goto unlock_cl;
197 	}
198 	list_add_tail(&cl->link, &dev->cl_list);
199 	set_bit(id, dev->host_clients_map);
200 	cl->state = ISHTP_CL_INITIALIZING;
201 
202 unlock_cl:
203 	spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl);
204 unlock_dev:
205 	spin_unlock_irqrestore(&dev->device_lock, flags);
206 	return ret;
207 }
208 EXPORT_SYMBOL(ishtp_cl_link);
209 
210 /**
211  * ishtp_cl_unlink() - remove fw_cl from the client device list
212  * @cl: client device instance
213  *
214  * Remove a previously linked device to a ishtp device
215  */
216 void ishtp_cl_unlink(struct ishtp_cl *cl)
217 {
218 	struct ishtp_device *dev;
219 	struct ishtp_cl *pos;
220 	unsigned long	flags;
221 
222 	/* don't shout on error exit path */
223 	if (!cl || !cl->dev)
224 		return;
225 
226 	dev = cl->dev;
227 
228 	spin_lock_irqsave(&dev->device_lock, flags);
229 	if (dev->open_handle_count > 0) {
230 		clear_bit(cl->host_client_id, dev->host_clients_map);
231 		dev->open_handle_count--;
232 	}
233 	spin_unlock_irqrestore(&dev->device_lock, flags);
234 
235 	/*
236 	 * This checks that 'cl' is actually linked into device's structure,
237 	 * before attempting 'list_del'
238 	 */
239 	spin_lock_irqsave(&dev->cl_list_lock, flags);
240 	list_for_each_entry(pos, &dev->cl_list, link)
241 		if (cl->host_client_id == pos->host_client_id) {
242 			list_del_init(&pos->link);
243 			break;
244 		}
245 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
246 }
247 EXPORT_SYMBOL(ishtp_cl_unlink);
248 
249 /**
250  * ishtp_cl_disconnect() - Send disconnect request to firmware
251  * @cl: client device instance
252  *
253  * Send a disconnect request for a client to firmware.
254  *
255  * Return: 0 if successful disconnect response from the firmware or error
256  * code on failure
257  */
258 int ishtp_cl_disconnect(struct ishtp_cl *cl)
259 {
260 	struct ishtp_device *dev;
261 	int err;
262 
263 	if (WARN_ON(!cl || !cl->dev))
264 		return -ENODEV;
265 
266 	dev = cl->dev;
267 
268 	dev->print_log(dev, "%s() state %d\n", __func__, cl->state);
269 
270 	if (cl->state != ISHTP_CL_DISCONNECTING) {
271 		dev->print_log(dev, "%s() Disconnect in progress\n", __func__);
272 		return 0;
273 	}
274 
275 	if (ishtp_hbm_cl_disconnect_req(dev, cl)) {
276 		dev->print_log(dev, "%s() Failed to disconnect\n", __func__);
277 		dev_err(&cl->device->dev, "failed to disconnect.\n");
278 		return -ENODEV;
279 	}
280 
281 	err = wait_event_interruptible_timeout(cl->wait_ctrl_res,
282 			(dev->dev_state != ISHTP_DEV_ENABLED ||
283 			cl->state == ISHTP_CL_DISCONNECTED),
284 			ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT));
285 
286 	/*
287 	 * If FW reset arrived, this will happen. Don't check cl->,
288 	 * as 'cl' may be freed already
289 	 */
290 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
291 		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
292 			       __func__);
293 		return -ENODEV;
294 	}
295 
296 	if (cl->state == ISHTP_CL_DISCONNECTED) {
297 		dev->print_log(dev, "%s() successful\n", __func__);
298 		return 0;
299 	}
300 
301 	return -ENODEV;
302 }
303 EXPORT_SYMBOL(ishtp_cl_disconnect);
304 
305 /**
306  * ishtp_cl_is_other_connecting() - Check other client is connecting
307  * @cl: client device instance
308  *
309  * Checks if other client with the same fw client id is connecting
310  *
311  * Return: true if other client is connected else false
312  */
313 static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl)
314 {
315 	struct ishtp_device *dev;
316 	struct ishtp_cl *pos;
317 	unsigned long	flags;
318 
319 	if (WARN_ON(!cl || !cl->dev))
320 		return false;
321 
322 	dev = cl->dev;
323 	spin_lock_irqsave(&dev->cl_list_lock, flags);
324 	list_for_each_entry(pos, &dev->cl_list, link) {
325 		if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) &&
326 				cl->fw_client_id == pos->fw_client_id) {
327 			spin_unlock_irqrestore(&dev->cl_list_lock, flags);
328 			return true;
329 		}
330 	}
331 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
332 
333 	return false;
334 }
335 
336 /**
337  * ishtp_cl_connect() - Send connect request to firmware
338  * @cl: client device instance
339  *
340  * Send a connect request for a client to firmware. If successful it will
341  * RX and TX ring buffers
342  *
343  * Return: 0 if successful connect response from the firmware and able
344  * to bind and allocate ring buffers or error code on failure
345  */
346 int ishtp_cl_connect(struct ishtp_cl *cl)
347 {
348 	struct ishtp_device *dev;
349 	int rets;
350 
351 	if (WARN_ON(!cl || !cl->dev))
352 		return -ENODEV;
353 
354 	dev = cl->dev;
355 
356 	dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state);
357 
358 	if (ishtp_cl_is_other_connecting(cl)) {
359 		dev->print_log(dev, "%s() Busy\n", __func__);
360 		return	-EBUSY;
361 	}
362 
363 	if (ishtp_hbm_cl_connect_req(dev, cl)) {
364 		dev->print_log(dev, "%s() HBM connect req fail\n", __func__);
365 		return -ENODEV;
366 	}
367 
368 	rets = wait_event_interruptible_timeout(cl->wait_ctrl_res,
369 				(dev->dev_state == ISHTP_DEV_ENABLED &&
370 				(cl->state == ISHTP_CL_CONNECTED ||
371 				 cl->state == ISHTP_CL_DISCONNECTED)),
372 				ishtp_secs_to_jiffies(
373 					ISHTP_CL_CONNECT_TIMEOUT));
374 	/*
375 	 * If FW reset arrived, this will happen. Don't check cl->,
376 	 * as 'cl' may be freed already
377 	 */
378 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
379 		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
380 			       __func__);
381 		return -EFAULT;
382 	}
383 
384 	if (cl->state != ISHTP_CL_CONNECTED) {
385 		dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n",
386 			       __func__);
387 		return -EFAULT;
388 	}
389 
390 	rets = cl->status;
391 	if (rets) {
392 		dev->print_log(dev, "%s() Invalid status\n", __func__);
393 		return rets;
394 	}
395 
396 	rets = ishtp_cl_device_bind(cl);
397 	if (rets) {
398 		dev->print_log(dev, "%s() Bind error\n", __func__);
399 		ishtp_cl_disconnect(cl);
400 		return rets;
401 	}
402 
403 	rets = ishtp_cl_alloc_rx_ring(cl);
404 	if (rets) {
405 		dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__);
406 		/* if failed allocation, disconnect */
407 		ishtp_cl_disconnect(cl);
408 		return rets;
409 	}
410 
411 	rets = ishtp_cl_alloc_tx_ring(cl);
412 	if (rets) {
413 		dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__);
414 		/* if failed allocation, disconnect */
415 		ishtp_cl_free_rx_ring(cl);
416 		ishtp_cl_disconnect(cl);
417 		return rets;
418 	}
419 
420 	/* Upon successful connection and allocation, emit flow-control */
421 	rets = ishtp_cl_read_start(cl);
422 
423 	dev->print_log(dev, "%s() successful\n", __func__);
424 
425 	return rets;
426 }
427 EXPORT_SYMBOL(ishtp_cl_connect);
428 
429 /**
430  * ishtp_cl_read_start() - Prepare to read client message
431  * @cl: client device instance
432  *
433  * Get a free buffer from pool of free read buffers and add to read buffer
434  * pool to add contents. Send a flow control request to firmware to be able
435  * send next message.
436  *
437  * Return: 0 if successful or error code on failure
438  */
439 int ishtp_cl_read_start(struct ishtp_cl *cl)
440 {
441 	struct ishtp_device *dev;
442 	struct ishtp_cl_rb *rb;
443 	int rets;
444 	int i;
445 	unsigned long	flags;
446 	unsigned long	dev_flags;
447 
448 	if (WARN_ON(!cl || !cl->dev))
449 		return -ENODEV;
450 
451 	dev = cl->dev;
452 
453 	if (cl->state != ISHTP_CL_CONNECTED)
454 		return -ENODEV;
455 
456 	if (dev->dev_state != ISHTP_DEV_ENABLED)
457 		return -ENODEV;
458 
459 	i = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
460 	if (i < 0) {
461 		dev_err(&cl->device->dev, "no such fw client %d\n",
462 			cl->fw_client_id);
463 		return -ENODEV;
464 	}
465 
466 	/* The current rb is the head of the free rb list */
467 	spin_lock_irqsave(&cl->free_list_spinlock, flags);
468 	if (list_empty(&cl->free_rb_list.list)) {
469 		dev_warn(&cl->device->dev,
470 			 "[ishtp-ish] Rx buffers pool is empty\n");
471 		rets = -ENOMEM;
472 		rb = NULL;
473 		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
474 		goto out;
475 	}
476 	rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list);
477 	list_del_init(&rb->list);
478 	spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
479 
480 	rb->cl = cl;
481 	rb->buf_idx = 0;
482 
483 	INIT_LIST_HEAD(&rb->list);
484 	rets = 0;
485 
486 	/*
487 	 * This must be BEFORE sending flow control -
488 	 * response in ISR may come too fast...
489 	 */
490 	spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
491 	list_add_tail(&rb->list, &dev->read_list.list);
492 	spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
493 	if (ishtp_hbm_cl_flow_control_req(dev, cl)) {
494 		rets = -ENODEV;
495 		goto out;
496 	}
497 out:
498 	/* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */
499 	if (rets && rb) {
500 		spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
501 		list_del(&rb->list);
502 		spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
503 
504 		spin_lock_irqsave(&cl->free_list_spinlock, flags);
505 		list_add_tail(&rb->list, &cl->free_rb_list.list);
506 		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
507 	}
508 	return rets;
509 }
510 
511 /**
512  * ishtp_cl_send() - Send a message to firmware
513  * @cl: client device instance
514  * @buf: message buffer
515  * @length: length of message
516  *
517  * If the client is correct state to send message, this function gets a buffer
518  * from tx ring buffers, copy the message data and call to send the message
519  * using ishtp_cl_send_msg()
520  *
521  * Return: 0 if successful or error code on failure
522  */
523 int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length)
524 {
525 	struct ishtp_device	*dev;
526 	int	id;
527 	struct ishtp_cl_tx_ring	*cl_msg;
528 	int	have_msg_to_send = 0;
529 	unsigned long	tx_flags, tx_free_flags;
530 
531 	if (WARN_ON(!cl || !cl->dev))
532 		return -ENODEV;
533 
534 	dev = cl->dev;
535 
536 	if (cl->state != ISHTP_CL_CONNECTED) {
537 		++cl->err_send_msg;
538 		return -EPIPE;
539 	}
540 
541 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
542 		++cl->err_send_msg;
543 		return -ENODEV;
544 	}
545 
546 	/* Check if we have fw client device */
547 	id = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
548 	if (id < 0) {
549 		++cl->err_send_msg;
550 		return -ENOENT;
551 	}
552 
553 	if (length > dev->fw_clients[id].props.max_msg_length) {
554 		++cl->err_send_msg;
555 		return -EMSGSIZE;
556 	}
557 
558 	/* No free bufs */
559 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
560 	if (list_empty(&cl->tx_free_list.list)) {
561 		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
562 			tx_free_flags);
563 		++cl->err_send_msg;
564 		return	-ENOMEM;
565 	}
566 
567 	cl_msg = list_first_entry(&cl->tx_free_list.list,
568 		struct ishtp_cl_tx_ring, list);
569 	if (!cl_msg->send_buf.data) {
570 		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
571 			tx_free_flags);
572 		return	-EIO;
573 		/* Should not happen, as free list is pre-allocated */
574 	}
575 	/*
576 	 * This is safe, as 'length' is already checked for not exceeding
577 	 * max ISHTP message size per client
578 	 */
579 	list_del_init(&cl_msg->list);
580 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
581 	memcpy(cl_msg->send_buf.data, buf, length);
582 	cl_msg->send_buf.size = length;
583 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
584 	have_msg_to_send = !list_empty(&cl->tx_list.list);
585 	list_add_tail(&cl_msg->list, &cl->tx_list.list);
586 	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
587 
588 	if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0)
589 		ishtp_cl_send_msg(dev, cl);
590 
591 	return	0;
592 }
593 EXPORT_SYMBOL(ishtp_cl_send);
594 
595 /**
596  * ishtp_cl_read_complete() - read complete
597  * @rb: Pointer to client request block
598  *
599  * If the message is completely received call ishtp_cl_bus_rx_event()
600  * to process message
601  */
602 static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb)
603 {
604 	unsigned long	flags;
605 	int	schedule_work_flag = 0;
606 	struct ishtp_cl	*cl = rb->cl;
607 
608 	spin_lock_irqsave(&cl->in_process_spinlock, flags);
609 	/*
610 	 * if in-process list is empty, then need to schedule
611 	 * the processing thread
612 	 */
613 	schedule_work_flag = list_empty(&cl->in_process_list.list);
614 	list_add_tail(&rb->list, &cl->in_process_list.list);
615 	spin_unlock_irqrestore(&cl->in_process_spinlock, flags);
616 
617 	if (schedule_work_flag)
618 		ishtp_cl_bus_rx_event(cl->device);
619 }
620 
621 /**
622  * ipc_tx_callback() - IPC tx callback function
623  * @prm: Pointer to client device instance
624  *
625  * Send message over IPC either first time or on callback on previous message
626  * completion
627  */
628 static void ipc_tx_callback(void *prm)
629 {
630 	struct ishtp_cl	*cl = prm;
631 	struct ishtp_cl_tx_ring	*cl_msg;
632 	size_t	rem;
633 	struct ishtp_device	*dev = (cl ? cl->dev : NULL);
634 	struct ishtp_msg_hdr	ishtp_hdr;
635 	unsigned long	tx_flags, tx_free_flags;
636 	unsigned char	*pmsg;
637 
638 	if (!dev)
639 		return;
640 
641 	/*
642 	 * Other conditions if some critical error has
643 	 * occurred before this callback is called
644 	 */
645 	if (dev->dev_state != ISHTP_DEV_ENABLED)
646 		return;
647 
648 	if (cl->state != ISHTP_CL_CONNECTED)
649 		return;
650 
651 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
652 	if (list_empty(&cl->tx_list.list)) {
653 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
654 		return;
655 	}
656 
657 	if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) {
658 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
659 		return;
660 	}
661 
662 	if (!cl->sending) {
663 		--cl->ishtp_flow_ctrl_creds;
664 		cl->last_ipc_acked = 0;
665 		cl->last_tx_path = CL_TX_PATH_IPC;
666 		cl->sending = 1;
667 	}
668 
669 	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
670 			    list);
671 	rem = cl_msg->send_buf.size - cl->tx_offs;
672 
673 	ishtp_hdr.host_addr = cl->host_client_id;
674 	ishtp_hdr.fw_addr = cl->fw_client_id;
675 	ishtp_hdr.reserved = 0;
676 	pmsg = cl_msg->send_buf.data + cl->tx_offs;
677 
678 	if (rem <= dev->mtu) {
679 		ishtp_hdr.length = rem;
680 		ishtp_hdr.msg_complete = 1;
681 		cl->sending = 0;
682 		list_del_init(&cl_msg->list);	/* Must be before write */
683 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
684 		/* Submit to IPC queue with no callback */
685 		ishtp_write_message(dev, &ishtp_hdr, pmsg);
686 		spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
687 		list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
688 		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
689 			tx_free_flags);
690 	} else {
691 		/* Send IPC fragment */
692 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
693 		cl->tx_offs += dev->mtu;
694 		ishtp_hdr.length = dev->mtu;
695 		ishtp_hdr.msg_complete = 0;
696 		ishtp_send_msg(dev, &ishtp_hdr, pmsg, ipc_tx_callback, cl);
697 	}
698 }
699 
700 /**
701  * ishtp_cl_send_msg_ipc() -Send message using IPC
702  * @dev: ISHTP device instance
703  * @cl: Pointer to client device instance
704  *
705  * Send message over IPC not using DMA
706  */
707 static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev,
708 				  struct ishtp_cl *cl)
709 {
710 	/* If last DMA message wasn't acked yet, leave this one in Tx queue */
711 	if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0)
712 		return;
713 
714 	cl->tx_offs = 0;
715 	ipc_tx_callback(cl);
716 	++cl->send_msg_cnt_ipc;
717 }
718 
719 /**
720  * ishtp_cl_send_msg_dma() -Send message using DMA
721  * @dev: ISHTP device instance
722  * @cl: Pointer to client device instance
723  *
724  * Send message using DMA
725  */
726 static void ishtp_cl_send_msg_dma(struct ishtp_device *dev,
727 	struct ishtp_cl *cl)
728 {
729 	struct ishtp_msg_hdr	hdr;
730 	struct dma_xfer_hbm	dma_xfer;
731 	unsigned char	*msg_addr;
732 	int off;
733 	struct ishtp_cl_tx_ring	*cl_msg;
734 	unsigned long tx_flags, tx_free_flags;
735 
736 	/* If last IPC message wasn't acked yet, leave this one in Tx queue */
737 	if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0)
738 		return;
739 
740 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
741 	if (list_empty(&cl->tx_list.list)) {
742 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
743 		return;
744 	}
745 
746 	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
747 		list);
748 
749 	msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size);
750 	if (!msg_addr) {
751 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
752 		if (dev->transfer_path == CL_TX_PATH_DEFAULT)
753 			ishtp_cl_send_msg_ipc(dev, cl);
754 		return;
755 	}
756 
757 	list_del_init(&cl_msg->list);	/* Must be before write */
758 	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
759 
760 	--cl->ishtp_flow_ctrl_creds;
761 	cl->last_dma_acked = 0;
762 	cl->last_dma_addr = msg_addr;
763 	cl->last_tx_path = CL_TX_PATH_DMA;
764 
765 	/* write msg to dma buf */
766 	memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size);
767 
768 	/* send dma_xfer hbm msg */
769 	off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf;
770 	ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm));
771 	dma_xfer.hbm = DMA_XFER;
772 	dma_xfer.fw_client_id = cl->fw_client_id;
773 	dma_xfer.host_client_id = cl->host_client_id;
774 	dma_xfer.reserved = 0;
775 	dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off;
776 	dma_xfer.msg_length = cl_msg->send_buf.size;
777 	dma_xfer.reserved2 = 0;
778 	ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer);
779 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
780 	list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
781 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
782 	++cl->send_msg_cnt_dma;
783 }
784 
785 /**
786  * ishtp_cl_send_msg() -Send message using DMA or IPC
787  * @dev: ISHTP device instance
788  * @cl: Pointer to client device instance
789  *
790  * Send message using DMA or IPC based on transfer_path
791  */
792 void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl)
793 {
794 	if (dev->transfer_path == CL_TX_PATH_DMA)
795 		ishtp_cl_send_msg_dma(dev, cl);
796 	else
797 		ishtp_cl_send_msg_ipc(dev, cl);
798 }
799 
800 /**
801  * recv_ishtp_cl_msg() -Receive client message
802  * @dev: ISHTP device instance
803  * @ishtp_hdr: Pointer to message header
804  *
805  * Receive and dispatch ISHTP client messages. This function executes in ISR
806  * or work queue context
807  */
808 void recv_ishtp_cl_msg(struct ishtp_device *dev,
809 		       struct ishtp_msg_hdr *ishtp_hdr)
810 {
811 	struct ishtp_cl *cl;
812 	struct ishtp_cl_rb *rb;
813 	struct ishtp_cl_rb *new_rb;
814 	unsigned char *buffer = NULL;
815 	struct ishtp_cl_rb *complete_rb = NULL;
816 	unsigned long	flags;
817 	int	rb_count;
818 
819 	if (ishtp_hdr->reserved) {
820 		dev_err(dev->devc, "corrupted message header.\n");
821 		goto	eoi;
822 	}
823 
824 	if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) {
825 		dev_err(dev->devc,
826 			"ISHTP message length in hdr exceeds IPC MTU\n");
827 		goto	eoi;
828 	}
829 
830 	spin_lock_irqsave(&dev->read_list_spinlock, flags);
831 	rb_count = -1;
832 	list_for_each_entry(rb, &dev->read_list.list, list) {
833 		++rb_count;
834 		cl = rb->cl;
835 		if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr &&
836 				cl->fw_client_id == ishtp_hdr->fw_addr) ||
837 				!(cl->state == ISHTP_CL_CONNECTED))
838 			continue;
839 
840 		 /* If no Rx buffer is allocated, disband the rb */
841 		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
842 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
843 			dev_err(&cl->device->dev,
844 				"Rx buffer is not allocated.\n");
845 			list_del(&rb->list);
846 			ishtp_io_rb_free(rb);
847 			cl->status = -ENOMEM;
848 			goto	eoi;
849 		}
850 
851 		/*
852 		 * If message buffer overflown (exceeds max. client msg
853 		 * size, drop message and return to free buffer.
854 		 * Do we need to disconnect such a client? (We don't send
855 		 * back FC, so communication will be stuck anyway)
856 		 */
857 		if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) {
858 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
859 			dev_err(&cl->device->dev,
860 				"message overflow. size %d len %d idx %ld\n",
861 				rb->buffer.size, ishtp_hdr->length,
862 				rb->buf_idx);
863 			list_del(&rb->list);
864 			ishtp_cl_io_rb_recycle(rb);
865 			cl->status = -EIO;
866 			goto	eoi;
867 		}
868 
869 		buffer = rb->buffer.data + rb->buf_idx;
870 		dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length);
871 
872 		rb->buf_idx += ishtp_hdr->length;
873 		if (ishtp_hdr->msg_complete) {
874 			/* Last fragment in message - it's complete */
875 			cl->status = 0;
876 			list_del(&rb->list);
877 			complete_rb = rb;
878 
879 			--cl->out_flow_ctrl_creds;
880 			/*
881 			 * the whole msg arrived, send a new FC, and add a new
882 			 * rb buffer for the next coming msg
883 			 */
884 			spin_lock(&cl->free_list_spinlock);
885 
886 			if (!list_empty(&cl->free_rb_list.list)) {
887 				new_rb = list_entry(cl->free_rb_list.list.next,
888 					struct ishtp_cl_rb, list);
889 				list_del_init(&new_rb->list);
890 				spin_unlock(&cl->free_list_spinlock);
891 				new_rb->cl = cl;
892 				new_rb->buf_idx = 0;
893 				INIT_LIST_HEAD(&new_rb->list);
894 				list_add_tail(&new_rb->list,
895 					&dev->read_list.list);
896 
897 				ishtp_hbm_cl_flow_control_req(dev, cl);
898 			} else {
899 				spin_unlock(&cl->free_list_spinlock);
900 			}
901 		}
902 		/* One more fragment in message (even if this was last) */
903 		++cl->recv_msg_num_frags;
904 
905 		/*
906 		 * We can safely break here (and in BH too),
907 		 * a single input message can go only to a single request!
908 		 */
909 		break;
910 	}
911 
912 	spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
913 	/* If it's nobody's message, just read and discard it */
914 	if (!buffer) {
915 		uint8_t	rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];
916 
917 		dev_err(dev->devc, "Dropped Rx msg - no request\n");
918 		dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);
919 		goto	eoi;
920 	}
921 
922 	if (complete_rb) {
923 		cl = complete_rb->cl;
924 		cl->ts_rx = ktime_get();
925 		++cl->recv_msg_cnt_ipc;
926 		ishtp_cl_read_complete(complete_rb);
927 	}
928 eoi:
929 	return;
930 }
931 
932 /**
933  * recv_ishtp_cl_msg_dma() -Receive client message
934  * @dev: ISHTP device instance
935  * @msg: message pointer
936  * @hbm: hbm buffer
937  *
938  * Receive and dispatch ISHTP client messages using DMA. This function executes
939  * in ISR or work queue context
940  */
941 void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg,
942 			   struct dma_xfer_hbm *hbm)
943 {
944 	struct ishtp_cl *cl;
945 	struct ishtp_cl_rb *rb;
946 	struct ishtp_cl_rb *new_rb;
947 	unsigned char *buffer = NULL;
948 	struct ishtp_cl_rb *complete_rb = NULL;
949 	unsigned long	flags;
950 
951 	spin_lock_irqsave(&dev->read_list_spinlock, flags);
952 
953 	list_for_each_entry(rb, &dev->read_list.list, list) {
954 		cl = rb->cl;
955 		if (!cl || !(cl->host_client_id == hbm->host_client_id &&
956 				cl->fw_client_id == hbm->fw_client_id) ||
957 				!(cl->state == ISHTP_CL_CONNECTED))
958 			continue;
959 
960 		/*
961 		 * If no Rx buffer is allocated, disband the rb
962 		 */
963 		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
964 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
965 			dev_err(&cl->device->dev,
966 				"response buffer is not allocated.\n");
967 			list_del(&rb->list);
968 			ishtp_io_rb_free(rb);
969 			cl->status = -ENOMEM;
970 			goto	eoi;
971 		}
972 
973 		/*
974 		 * If message buffer overflown (exceeds max. client msg
975 		 * size, drop message and return to free buffer.
976 		 * Do we need to disconnect such a client? (We don't send
977 		 * back FC, so communication will be stuck anyway)
978 		 */
979 		if (rb->buffer.size < hbm->msg_length) {
980 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
981 			dev_err(&cl->device->dev,
982 				"message overflow. size %d len %d idx %ld\n",
983 				rb->buffer.size, hbm->msg_length, rb->buf_idx);
984 			list_del(&rb->list);
985 			ishtp_cl_io_rb_recycle(rb);
986 			cl->status = -EIO;
987 			goto	eoi;
988 		}
989 
990 		buffer = rb->buffer.data;
991 		memcpy(buffer, msg, hbm->msg_length);
992 		rb->buf_idx = hbm->msg_length;
993 
994 		/* Last fragment in message - it's complete */
995 		cl->status = 0;
996 		list_del(&rb->list);
997 		complete_rb = rb;
998 
999 		--cl->out_flow_ctrl_creds;
1000 		/*
1001 		 * the whole msg arrived, send a new FC, and add a new
1002 		 * rb buffer for the next coming msg
1003 		 */
1004 		spin_lock(&cl->free_list_spinlock);
1005 
1006 		if (!list_empty(&cl->free_rb_list.list)) {
1007 			new_rb = list_entry(cl->free_rb_list.list.next,
1008 				struct ishtp_cl_rb, list);
1009 			list_del_init(&new_rb->list);
1010 			spin_unlock(&cl->free_list_spinlock);
1011 			new_rb->cl = cl;
1012 			new_rb->buf_idx = 0;
1013 			INIT_LIST_HEAD(&new_rb->list);
1014 			list_add_tail(&new_rb->list,
1015 				&dev->read_list.list);
1016 
1017 			ishtp_hbm_cl_flow_control_req(dev, cl);
1018 		} else {
1019 			spin_unlock(&cl->free_list_spinlock);
1020 		}
1021 
1022 		/* One more fragment in message (this is always last) */
1023 		++cl->recv_msg_num_frags;
1024 
1025 		/*
1026 		 * We can safely break here (and in BH too),
1027 		 * a single input message can go only to a single request!
1028 		 */
1029 		break;
1030 	}
1031 
1032 	spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1033 	/* If it's nobody's message, just read and discard it */
1034 	if (!buffer) {
1035 		dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n");
1036 		goto	eoi;
1037 	}
1038 
1039 	if (complete_rb) {
1040 		cl = complete_rb->cl;
1041 		cl->ts_rx = ktime_get();
1042 		++cl->recv_msg_cnt_dma;
1043 		ishtp_cl_read_complete(complete_rb);
1044 	}
1045 eoi:
1046 	return;
1047 }
1048