71 static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,  in tegra_ivc_invalidate_counter()  argument
79 static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,  in tegra_ivc_flush_counter()  argument
87 static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,  in tegra_ivc_frame_addr()  argument
91 	BUG_ON(frame >= ivc->nframes);  in tegra_ivc_frame_addr()
94 	       (ivc->frame_size * frame);  in tegra_ivc_frame_addr()
97 static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,  in tegra_ivc_frame_pointer()  argument
101 	return (void *)tegra_ivc_frame_addr(ivc, ch, frame);  in tegra_ivc_frame_pointer()
104 static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,  in tegra_ivc_invalidate_frame()  argument
108 	ulong base = tegra_ivc_frame_addr(ivc, h, frame);  in tegra_ivc_invalidate_frame()
109 	invalidate_dcache_range(base, base + ivc->frame_size);  in tegra_ivc_invalidate_frame()
112 static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,  in tegra_ivc_flush_frame()  argument
116 	ulong base = tegra_ivc_frame_addr(ivc, h, frame);  in tegra_ivc_flush_frame()
117 	flush_dcache_range(base, base + ivc->frame_size);  in tegra_ivc_flush_frame()
120 static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,  in tegra_ivc_channel_empty()  argument
141 	if (w_count - r_count > ivc->nframes)  in tegra_ivc_channel_empty()
147 static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,  in tegra_ivc_channel_full()  argument
155 	       ivc->nframes;  in tegra_ivc_channel_full()
158 static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)  in tegra_ivc_advance_rx()  argument
160 	ACCESS_ONCE(ivc->rx_channel->r_count) =  in tegra_ivc_advance_rx()
161 			ACCESS_ONCE(ivc->rx_channel->r_count) + 1;  in tegra_ivc_advance_rx()
163 	if (ivc->r_pos == ivc->nframes - 1)  in tegra_ivc_advance_rx()
164 		ivc->r_pos = 0;  in tegra_ivc_advance_rx()
166 		ivc->r_pos++;  in tegra_ivc_advance_rx()
169 static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)  in tegra_ivc_advance_tx()  argument
171 	ACCESS_ONCE(ivc->tx_channel->w_count) =  in tegra_ivc_advance_tx()
172 			ACCESS_ONCE(ivc->tx_channel->w_count) + 1;  in tegra_ivc_advance_tx()
174 	if (ivc->w_pos == ivc->nframes - 1)  in tegra_ivc_advance_tx()
175 		ivc->w_pos = 0;  in tegra_ivc_advance_tx()
177 		ivc->w_pos++;  in tegra_ivc_advance_tx()
180 static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)  in tegra_ivc_check_read()  argument
192 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_check_read()
201 	if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))  in tegra_ivc_check_read()
205 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_check_read()
206 	return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;  in tegra_ivc_check_read()
209 static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)  in tegra_ivc_check_write()  argument
213 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_check_write()
216 	if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))  in tegra_ivc_check_write()
220 	tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_check_write()
221 	return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;  in tegra_ivc_check_write()
224 static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,  in tegra_ivc_channel_avail_count()  argument
236 int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)  in tegra_ivc_read_get_next_frame()  argument
238 	int result = tegra_ivc_check_read(ivc);  in tegra_ivc_read_get_next_frame()
248 	tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);  in tegra_ivc_read_get_next_frame()
249 	*frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);  in tegra_ivc_read_get_next_frame()
254 int tegra_ivc_read_advance(struct tegra_ivc *ivc)  in tegra_ivc_read_advance()  argument
264 	result = tegra_ivc_check_read(ivc);  in tegra_ivc_read_advance()
268 	tegra_ivc_advance_rx(ivc);  in tegra_ivc_read_advance()
270 	tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_read_advance()
278 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_read_advance()
280 	if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==  in tegra_ivc_read_advance()
281 	    ivc->nframes - 1)  in tegra_ivc_read_advance()
282 		ivc->notify(ivc);  in tegra_ivc_read_advance()
287 int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)  in tegra_ivc_write_get_next_frame()  argument
289 	int result = tegra_ivc_check_write(ivc);  in tegra_ivc_write_get_next_frame()
293 	*frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);  in tegra_ivc_write_get_next_frame()
298 int tegra_ivc_write_advance(struct tegra_ivc *ivc)  in tegra_ivc_write_advance()  argument
303 	result = tegra_ivc_check_write(ivc);  in tegra_ivc_write_advance()
307 	tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);  in tegra_ivc_write_advance()
314 	tegra_ivc_advance_tx(ivc);  in tegra_ivc_write_advance()
316 	tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_write_advance()
324 	tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_write_advance()
326 	if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)  in tegra_ivc_write_advance()
327 		ivc->notify(ivc);  in tegra_ivc_write_advance()
351 int tegra_ivc_channel_notified(struct tegra_ivc *ivc)  in tegra_ivc_channel_notified()  argument
358 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_channel_notified()
359 	peer_state = ACCESS_ONCE(ivc->rx_channel->state);  in tegra_ivc_channel_notified()
373 		ivc->tx_channel->w_count = 0;  in tegra_ivc_channel_notified()
374 		ivc->rx_channel->r_count = 0;  in tegra_ivc_channel_notified()
376 		ivc->w_pos = 0;  in tegra_ivc_channel_notified()
377 		ivc->r_pos = 0;  in tegra_ivc_channel_notified()
389 		ivc->tx_channel->state = ivc_state_ack;  in tegra_ivc_channel_notified()
391 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
396 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
397 	} else if (ivc->tx_channel->state == ivc_state_sync &&  in tegra_ivc_channel_notified()
410 		ivc->tx_channel->w_count = 0;  in tegra_ivc_channel_notified()
411 		ivc->rx_channel->r_count = 0;  in tegra_ivc_channel_notified()
413 		ivc->w_pos = 0;  in tegra_ivc_channel_notified()
414 		ivc->r_pos = 0;  in tegra_ivc_channel_notified()
427 		ivc->tx_channel->state = ivc_state_established;  in tegra_ivc_channel_notified()
429 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
434 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
435 	} else if (ivc->tx_channel->state == ivc_state_ack) {  in tegra_ivc_channel_notified()
449 		ivc->tx_channel->state = ivc_state_established;  in tegra_ivc_channel_notified()
451 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
456 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
466 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_channel_notified()
472 void tegra_ivc_channel_reset(struct tegra_ivc *ivc)  in tegra_ivc_channel_reset()  argument
476 	ivc->tx_channel->state = ivc_state_sync;  in tegra_ivc_channel_reset()
478 	tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_reset()
479 	ivc->notify(ivc);  in tegra_ivc_channel_reset()
530 int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,  in tegra_ivc_init()  argument
536 	if (!ivc)  in tegra_ivc_init()
543 	ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;  in tegra_ivc_init()
544 	ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;  in tegra_ivc_init()
545 	ivc->w_pos = 0;  in tegra_ivc_init()
546 	ivc->r_pos = 0;  in tegra_ivc_init()
547 	ivc->nframes = nframes;  in tegra_ivc_init()
548 	ivc->frame_size = frame_size;  in tegra_ivc_init()
549 	ivc->notify = notify;  in tegra_ivc_init()