77 static inline void tegra_ivc_invalidate(struct tegra_ivc *ivc, dma_addr_t phys)  in tegra_ivc_invalidate()  argument
79 	if (!ivc->peer)  in tegra_ivc_invalidate()
82 	dma_sync_single_for_cpu(ivc->peer, phys, TEGRA_IVC_ALIGN,  in tegra_ivc_invalidate()
86 static inline void tegra_ivc_flush(struct tegra_ivc *ivc, dma_addr_t phys)  in tegra_ivc_flush()  argument
88 	if (!ivc->peer)  in tegra_ivc_flush()
91 	dma_sync_single_for_device(ivc->peer, phys, TEGRA_IVC_ALIGN,  in tegra_ivc_flush()
95 static inline bool tegra_ivc_empty(struct tegra_ivc *ivc, struct iosys_map *map)  in tegra_ivc_empty()  argument
115 	if (tx - rx > ivc->num_frames)  in tegra_ivc_empty()
121 static inline bool tegra_ivc_full(struct tegra_ivc *ivc, struct iosys_map *map)  in tegra_ivc_full()  argument
130 	return tx - rx >= ivc->num_frames;  in tegra_ivc_full()
133 static inline u32 tegra_ivc_available(struct tegra_ivc *ivc, struct iosys_map *map)  in tegra_ivc_available()  argument
147 static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)  in tegra_ivc_advance_tx()  argument
149 	unsigned int count = tegra_ivc_header_read_field(&ivc->tx.map, tx.count);  in tegra_ivc_advance_tx()
151 	tegra_ivc_header_write_field(&ivc->tx.map, tx.count, count + 1);  in tegra_ivc_advance_tx()
153 	if (ivc->tx.position == ivc->num_frames - 1)  in tegra_ivc_advance_tx()
154 		ivc->tx.position = 0;  in tegra_ivc_advance_tx()
156 		ivc->tx.position++;  in tegra_ivc_advance_tx()
159 static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)  in tegra_ivc_advance_rx()  argument
161 	unsigned int count = tegra_ivc_header_read_field(&ivc->rx.map, rx.count);  in tegra_ivc_advance_rx()
163 	tegra_ivc_header_write_field(&ivc->rx.map, rx.count, count + 1);  in tegra_ivc_advance_rx()
165 	if (ivc->rx.position == ivc->num_frames - 1)  in tegra_ivc_advance_rx()
166 		ivc->rx.position = 0;  in tegra_ivc_advance_rx()
168 		ivc->rx.position++;  in tegra_ivc_advance_rx()
171 static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)  in tegra_ivc_check_read()  argument
184 	state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);  in tegra_ivc_check_read()
195 	if (!tegra_ivc_empty(ivc, &ivc->rx.map))  in tegra_ivc_check_read()
198 	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);  in tegra_ivc_check_read()
200 	if (tegra_ivc_empty(ivc, &ivc->rx.map))  in tegra_ivc_check_read()
206 static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)  in tegra_ivc_check_write()  argument
211 	state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);  in tegra_ivc_check_write()
215 	if (!tegra_ivc_full(ivc, &ivc->tx.map))  in tegra_ivc_check_write()
218 	tegra_ivc_invalidate(ivc, ivc->tx.phys + offset);  in tegra_ivc_check_write()
220 	if (tegra_ivc_full(ivc, &ivc->tx.map))  in tegra_ivc_check_write()
226 static int tegra_ivc_frame_virt(struct tegra_ivc *ivc, const struct iosys_map *header,  in tegra_ivc_frame_virt()  argument
229 	size_t offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;  in tegra_ivc_frame_virt()
231 	if (WARN_ON(frame >= ivc->num_frames))  in tegra_ivc_frame_virt()
239 static inline dma_addr_t tegra_ivc_frame_phys(struct tegra_ivc *ivc,  in tegra_ivc_frame_phys()  argument
245 	offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;  in tegra_ivc_frame_phys()
250 static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,  in tegra_ivc_invalidate_frame()  argument
256 	if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))  in tegra_ivc_invalidate_frame()
259 	phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;  in tegra_ivc_invalidate_frame()
261 	dma_sync_single_for_cpu(ivc->peer, phys, size, DMA_FROM_DEVICE);  in tegra_ivc_invalidate_frame()
264 static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,  in tegra_ivc_flush_frame()  argument
270 	if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))  in tegra_ivc_flush_frame()
273 	phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;  in tegra_ivc_flush_frame()
275 	dma_sync_single_for_device(ivc->peer, phys, size, DMA_TO_DEVICE);  in tegra_ivc_flush_frame()
279 int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, struct iosys_map *map)  in tegra_ivc_read_get_next_frame()  argument
283 	if (WARN_ON(ivc == NULL))  in tegra_ivc_read_get_next_frame()
286 	err = tegra_ivc_check_read(ivc);  in tegra_ivc_read_get_next_frame()
296 	tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0,  in tegra_ivc_read_get_next_frame()
297 				   ivc->frame_size);  in tegra_ivc_read_get_next_frame()
299 	return tegra_ivc_frame_virt(ivc, &ivc->rx.map, ivc->rx.position, map);  in tegra_ivc_read_get_next_frame()
303 int tegra_ivc_read_advance(struct tegra_ivc *ivc)  in tegra_ivc_read_advance()  argument
314 	err = tegra_ivc_check_read(ivc);  in tegra_ivc_read_advance()
318 	tegra_ivc_advance_rx(ivc);  in tegra_ivc_read_advance()
320 	tegra_ivc_flush(ivc, ivc->rx.phys + rx);  in tegra_ivc_read_advance()
333 	tegra_ivc_invalidate(ivc, ivc->rx.phys + tx);  in tegra_ivc_read_advance()
335 	if (tegra_ivc_available(ivc, &ivc->rx.map) == ivc->num_frames - 1)  in tegra_ivc_read_advance()
336 		ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_read_advance()
343 int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, struct iosys_map *map)  in tegra_ivc_write_get_next_frame()  argument
347 	err = tegra_ivc_check_write(ivc);  in tegra_ivc_write_get_next_frame()
351 	return tegra_ivc_frame_virt(ivc, &ivc->tx.map, ivc->tx.position, map);  in tegra_ivc_write_get_next_frame()
356 int tegra_ivc_write_advance(struct tegra_ivc *ivc)  in tegra_ivc_write_advance()  argument
362 	err = tegra_ivc_check_write(ivc);  in tegra_ivc_write_advance()
366 	tegra_ivc_flush_frame(ivc, ivc->tx.phys, ivc->tx.position, 0,  in tegra_ivc_write_advance()
367 			      ivc->frame_size);  in tegra_ivc_write_advance()
375 	tegra_ivc_advance_tx(ivc);  in tegra_ivc_write_advance()
376 	tegra_ivc_flush(ivc, ivc->tx.phys + tx);  in tegra_ivc_write_advance()
389 	tegra_ivc_invalidate(ivc, ivc->tx.phys + rx);  in tegra_ivc_write_advance()
391 	if (tegra_ivc_available(ivc, &ivc->tx.map) == 1)  in tegra_ivc_write_advance()
392 		ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_write_advance()
398 void tegra_ivc_reset(struct tegra_ivc *ivc)  in tegra_ivc_reset()  argument
402 	tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_SYNC);  in tegra_ivc_reset()
403 	tegra_ivc_flush(ivc, ivc->tx.phys + offset);  in tegra_ivc_reset()
404 	ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_reset()
428 int tegra_ivc_notified(struct tegra_ivc *ivc)  in tegra_ivc_notified()  argument
434 	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);  in tegra_ivc_notified()
435 	rx_state = tegra_ivc_header_read_field(&ivc->rx.map, tx.state);  in tegra_ivc_notified()
436 	tx_state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);  in tegra_ivc_notified()
452 		tegra_ivc_header_write_field(&ivc->tx.map, tx.count, 0);  in tegra_ivc_notified()
453 		tegra_ivc_header_write_field(&ivc->rx.map, rx.count, 0);  in tegra_ivc_notified()
455 		ivc->tx.position = 0;  in tegra_ivc_notified()
456 		ivc->rx.position = 0;  in tegra_ivc_notified()
468 		tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ACK);  in tegra_ivc_notified()
469 		tegra_ivc_flush(ivc, ivc->tx.phys + offset);  in tegra_ivc_notified()
474 		ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_notified()
491 		tegra_ivc_header_write_field(&ivc->tx.map, tx.count, 0);  in tegra_ivc_notified()
492 		tegra_ivc_header_write_field(&ivc->rx.map, rx.count, 0);  in tegra_ivc_notified()
494 		ivc->tx.position = 0;  in tegra_ivc_notified()
495 		ivc->rx.position = 0;  in tegra_ivc_notified()
508 		tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ESTABLISHED);  in tegra_ivc_notified()
509 		tegra_ivc_flush(ivc, ivc->tx.phys + offset);  in tegra_ivc_notified()
514 		ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_notified()
532 		tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ESTABLISHED);  in tegra_ivc_notified()
533 		tegra_ivc_flush(ivc, ivc->tx.phys + offset);  in tegra_ivc_notified()
538 		ivc->notify(ivc, ivc->notify_data);  in tegra_ivc_notified()
646 int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, const struct iosys_map *rx,  in tegra_ivc_init()  argument
649 		   void (*notify)(struct tegra_ivc *ivc, void *data),  in tegra_ivc_init()  argument
655 	if (WARN_ON(!ivc || !notify))  in tegra_ivc_init()
673 		ivc->rx.phys = dma_map_single(peer, iosys_map_get_vaddr(rx), queue_size,  in tegra_ivc_init()
675 		if (dma_mapping_error(peer, ivc->rx.phys))  in tegra_ivc_init()
678 		ivc->tx.phys = dma_map_single(peer, iosys_map_get_vaddr(tx), queue_size,  in tegra_ivc_init()
680 		if (dma_mapping_error(peer, ivc->tx.phys)) {  in tegra_ivc_init()
681 			dma_unmap_single(peer, ivc->rx.phys, queue_size,  in tegra_ivc_init()
686 		ivc->rx.phys = rx_phys;  in tegra_ivc_init()
687 		ivc->tx.phys = tx_phys;  in tegra_ivc_init()
690 	iosys_map_copy(&ivc->rx.map, rx);  in tegra_ivc_init()
691 	iosys_map_copy(&ivc->tx.map, tx);  in tegra_ivc_init()
692 	ivc->peer = peer;  in tegra_ivc_init()
693 	ivc->notify = notify;  in tegra_ivc_init()
694 	ivc->notify_data = data;  in tegra_ivc_init()
695 	ivc->frame_size = frame_size;  in tegra_ivc_init()
696 	ivc->num_frames = num_frames;  in tegra_ivc_init()
702 	ivc->tx.position = 0;  in tegra_ivc_init()
703 	ivc->rx.position = 0;  in tegra_ivc_init()
709 void tegra_ivc_cleanup(struct tegra_ivc *ivc)  in tegra_ivc_cleanup()  argument
711 	if (ivc->peer) {  in tegra_ivc_cleanup()
712 		size_t size = tegra_ivc_total_queue_size(ivc->num_frames *  in tegra_ivc_cleanup()
713 							 ivc->frame_size);  in tegra_ivc_cleanup()
715 		dma_unmap_single(ivc->peer, ivc->rx.phys, size,  in tegra_ivc_cleanup()
717 		dma_unmap_single(ivc->peer, ivc->tx.phys, size,  in tegra_ivc_cleanup()