Lines Matching +full:dma +full:- +full:pool
1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2022 Linaro Ltd.
12 #include <linux/dma-direction.h>
44 * transaction. If a buffer in an operation must be mapped for DMA, this is
75 __le64 addr; /* DMA address */
81 /* gsi_tre->flags mask values (in CPU byte order) */
87 int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count, in gsi_trans_pool_init() argument
94 return -EINVAL; in gsi_trans_pool_init()
96 return -EINVAL; in gsi_trans_pool_init()
98 return -EINVAL; in gsi_trans_pool_init()
100 /* By allocating a few extra entries in our pool (one less in gsi_trans_pool_init()
103 * ever worrying about straddling the end of the pool array. in gsi_trans_pool_init()
105 * we just allocate free entries from the beginning of the pool. in gsi_trans_pool_init()
107 alloc_size = size_mul(count + max_alloc - 1, size); in gsi_trans_pool_init()
111 return -ENOMEM; in gsi_trans_pool_init()
113 pool->base = virt; in gsi_trans_pool_init()
115 pool->count = alloc_size / size; in gsi_trans_pool_init()
116 pool->free = 0; in gsi_trans_pool_init()
117 pool->max_alloc = max_alloc; in gsi_trans_pool_init()
118 pool->size = size; in gsi_trans_pool_init()
119 pool->addr = 0; /* Only used for DMA pools */ in gsi_trans_pool_init()
124 void gsi_trans_pool_exit(struct gsi_trans_pool *pool) in gsi_trans_pool_exit() argument
126 kfree(pool->base); in gsi_trans_pool_exit()
127 memset(pool, 0, sizeof(*pool)); in gsi_trans_pool_exit()
130 /* Home-grown DMA pool. This way we can preallocate the pool, and guarantee
132 * require up to max_alloc elements from the pool. But we only allow
133 * allocation of a single element from a DMA pool at a time.
135 int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool, in gsi_trans_pool_init_dma() argument
143 return -EINVAL; in gsi_trans_pool_init_dma()
145 return -EINVAL; in gsi_trans_pool_init_dma()
147 return -EINVAL; in gsi_trans_pool_init_dma()
149 /* Don't let allocations cross a power-of-two boundary */ in gsi_trans_pool_init_dma()
151 total_size = (count + max_alloc - 1) * size; in gsi_trans_pool_init_dma()
153 /* The allocator will give us a power-of-2 number of pages in gsi_trans_pool_init_dma()
163 return -ENOMEM; in gsi_trans_pool_init_dma()
165 pool->base = virt; in gsi_trans_pool_init_dma()
166 pool->count = total_size / size; in gsi_trans_pool_init_dma()
167 pool->free = 0; in gsi_trans_pool_init_dma()
168 pool->size = size; in gsi_trans_pool_init_dma()
169 pool->max_alloc = max_alloc; in gsi_trans_pool_init_dma()
170 pool->addr = addr; in gsi_trans_pool_init_dma()
175 void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool) in gsi_trans_pool_exit_dma() argument
177 size_t total_size = pool->count * pool->size; in gsi_trans_pool_exit_dma()
179 dma_free_coherent(dev, total_size, pool->base, pool->addr); in gsi_trans_pool_exit_dma()
180 memset(pool, 0, sizeof(*pool)); in gsi_trans_pool_exit_dma()
183 /* Return the byte offset of the next free entry in the pool */
184 static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count) in gsi_trans_pool_alloc_common() argument
189 WARN_ON(count > pool->max_alloc); in gsi_trans_pool_alloc_common()
192 if (count > pool->count - pool->free) in gsi_trans_pool_alloc_common()
193 pool->free = 0; in gsi_trans_pool_alloc_common()
195 offset = pool->free * pool->size; in gsi_trans_pool_alloc_common()
196 pool->free += count; in gsi_trans_pool_alloc_common()
197 memset(pool->base + offset, 0, count * pool->size); in gsi_trans_pool_alloc_common()
202 /* Allocate a contiguous block of zeroed entries from a pool */
203 void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count) in gsi_trans_pool_alloc() argument
205 return pool->base + gsi_trans_pool_alloc_common(pool, count); in gsi_trans_pool_alloc()
208 /* Allocate a single zeroed entry from a DMA pool */
209 void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr) in gsi_trans_pool_alloc_dma() argument
211 u32 offset = gsi_trans_pool_alloc_common(pool, 1); in gsi_trans_pool_alloc_dma()
213 *addr = pool->addr + offset; in gsi_trans_pool_alloc_dma()
215 return pool->base + offset; in gsi_trans_pool_alloc_dma()
221 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_map()
224 index += trans->used_count - 1; in gsi_trans_map()
227 channel->trans_info.map[index % channel->tre_ring.count] = trans; in gsi_trans_map()
235 return channel->trans_info.map[index % channel->tre_ring.count]; in gsi_channel_trans_mapped()
241 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_complete()
242 u16 trans_id = trans_info->completed_id; in gsi_channel_trans_complete()
244 if (trans_id == trans_info->pending_id) { in gsi_channel_trans_complete()
246 if (trans_id == trans_info->pending_id) in gsi_channel_trans_complete()
250 return &trans_info->trans[trans_id %= channel->tre_count]; in gsi_channel_trans_complete()
256 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_committed()
257 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_committed()
260 trans_info->allocated_id++; in gsi_trans_move_committed()
266 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_pending()
267 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_pending()
268 u16 trans_index = trans - &trans_info->trans[0]; in gsi_trans_move_pending()
272 delta = trans_index - trans_info->committed_id + 1; in gsi_trans_move_pending()
273 trans_info->committed_id += delta % channel->tre_count; in gsi_trans_move_pending()
279 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_complete()
280 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_complete()
281 u16 trans_index = trans - trans_info->trans; in gsi_trans_move_complete()
285 delta = trans_index - trans_info->pending_id + 1; in gsi_trans_move_complete()
286 delta %= channel->tre_count; in gsi_trans_move_complete()
287 trans_info->pending_id += delta; in gsi_trans_move_complete()
293 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_polled()
294 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_polled()
297 trans_info->completed_id++; in gsi_trans_move_polled()
304 int avail = atomic_read(&trans_info->tre_avail); in gsi_trans_tre_reserve()
308 new = avail - (int)tre_count; in gsi_trans_tre_reserve()
311 } while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new)); in gsi_trans_tre_reserve()
316 /* Release previously-reserved TRE entries to a channel */
320 atomic_add(tre_count, &trans_info->tre_avail); in gsi_trans_tre_release()
329 trans_info = &gsi->channel[channel_id].trans_info; in gsi_channel_trans_idle()
331 return atomic_read(&trans_info->tre_avail) == tre_max; in gsi_channel_trans_idle()
339 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_alloc()
344 if (WARN_ON(tre_count > channel->trans_tre_max)) in gsi_channel_trans_alloc()
347 trans_info = &channel->trans_info; in gsi_channel_trans_alloc()
353 trans_index = trans_info->free_id % channel->tre_count; in gsi_channel_trans_alloc()
354 trans = &trans_info->trans[trans_index]; in gsi_channel_trans_alloc()
357 /* Initialize non-zero fields in the transaction */ in gsi_channel_trans_alloc()
358 trans->gsi = gsi; in gsi_channel_trans_alloc()
359 trans->channel_id = channel_id; in gsi_channel_trans_alloc()
360 trans->rsvd_count = tre_count; in gsi_channel_trans_alloc()
361 init_completion(&trans->completion); in gsi_channel_trans_alloc()
364 trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); in gsi_channel_trans_alloc()
365 sg_init_marker(trans->sgl, tre_count); in gsi_channel_trans_alloc()
367 trans->direction = direction; in gsi_channel_trans_alloc()
368 refcount_set(&trans->refcount, 1); in gsi_channel_trans_alloc()
371 trans_info->free_id++; in gsi_channel_trans_alloc()
376 /* Free a previously-allocated transaction */
381 if (!refcount_dec_and_test(&trans->refcount)) in gsi_trans_free()
387 trans_info = &trans->gsi->channel[trans->channel_id].trans_info; in gsi_trans_free()
388 if (!trans->used_count) { in gsi_trans_free()
389 trans_info->allocated_id++; in gsi_trans_free()
390 trans_info->committed_id++; in gsi_trans_free()
391 trans_info->pending_id++; in gsi_trans_free()
392 trans_info->completed_id++; in gsi_trans_free()
398 trans_info->polled_id++; in gsi_trans_free()
403 gsi_trans_tre_release(trans_info, trans->rsvd_count); in gsi_trans_free()
410 u32 which = trans->used_count++; in gsi_trans_cmd_add()
413 WARN_ON(which >= trans->rsvd_count); in gsi_trans_cmd_add()
416 * Their payloads come from a pool whose memory is allocated in gsi_trans_cmd_add()
418 * for DMA (unlike what we do for pages and skbs). in gsi_trans_cmd_add()
425 * entry are the DMA address and length. We still need the SG in gsi_trans_cmd_add()
429 sg = &trans->sgl[which]; in gsi_trans_cmd_add()
434 trans->cmd_opcode[which] = opcode; in gsi_trans_cmd_add()
441 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_page_add()
444 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_page_add()
445 return -EINVAL; in gsi_trans_page_add()
446 if (WARN_ON(trans->used_count)) in gsi_trans_page_add()
447 return -EINVAL; in gsi_trans_page_add()
450 ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); in gsi_trans_page_add()
452 return -ENOMEM; in gsi_trans_page_add()
454 trans->used_count++; /* Transaction now owns the (DMA mapped) page */ in gsi_trans_page_add()
462 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_skb_add()
466 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_skb_add()
467 return -EINVAL; in gsi_trans_skb_add()
468 if (WARN_ON(trans->used_count)) in gsi_trans_skb_add()
469 return -EINVAL; in gsi_trans_skb_add()
471 /* skb->len will not be 0 (checked early) */ in gsi_trans_skb_add()
472 ret = skb_to_sgvec(skb, sg, 0, skb->len); in gsi_trans_skb_add()
477 ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction); in gsi_trans_skb_add()
479 return -ENOMEM; in gsi_trans_skb_add()
481 /* Transaction now owns the (DMA mapped) skb */ in gsi_trans_skb_add()
482 trans->used_count += used_count; in gsi_trans_skb_add()
535 * __gsi_trans_commit() - Common GSI transaction commit code
547 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in __gsi_trans_commit()
548 struct gsi_ring *tre_ring = &channel->tre_ring; in __gsi_trans_commit()
550 bool bei = channel->toward_ipa; in __gsi_trans_commit()
558 WARN_ON(!trans->used_count); in __gsi_trans_commit()
565 cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL; in __gsi_trans_commit()
566 avail = tre_ring->count - tre_ring->index % tre_ring->count; in __gsi_trans_commit()
567 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in __gsi_trans_commit()
568 for_each_sg(trans->sgl, sg, trans->used_count, i) { in __gsi_trans_commit()
569 bool last_tre = i == trans->used_count - 1; in __gsi_trans_commit()
574 if (!avail--) in __gsi_trans_commit()
583 gsi_trans_map(trans, tre_ring->index); in __gsi_trans_commit()
585 tre_ring->index += trans->used_count; in __gsi_trans_commit()
587 trans->len = byte_count; in __gsi_trans_commit()
588 if (channel->toward_ipa) in __gsi_trans_commit()
594 if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { in __gsi_trans_commit()
596 if (channel->toward_ipa) in __gsi_trans_commit()
606 if (trans->used_count) in gsi_trans_commit()
615 if (!trans->used_count) in gsi_trans_commit_wait()
618 refcount_inc(&trans->refcount); in gsi_trans_commit_wait()
622 wait_for_completion(&trans->completion); in gsi_trans_commit_wait()
632 if (trans->direction != DMA_NONE) in gsi_trans_complete()
633 dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count, in gsi_trans_complete()
634 trans->direction); in gsi_trans_complete()
638 complete(&trans->completion); in gsi_trans_complete()
646 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_cancel_pending()
647 u16 trans_id = trans_info->pending_id; in gsi_channel_trans_cancel_pending()
649 /* channel->gsi->mutex is held by caller */ in gsi_channel_trans_cancel_pending()
652 if (trans_id == trans_info->committed_id) in gsi_channel_trans_cancel_pending()
659 trans = &trans_info->trans[trans_id % channel->tre_count]; in gsi_channel_trans_cancel_pending()
660 trans->cancelled = true; in gsi_channel_trans_cancel_pending()
661 } while (++trans_id != trans_info->committed_id); in gsi_channel_trans_cancel_pending()
664 trans_info->pending_id = trans_info->committed_id; in gsi_channel_trans_cancel_pending()
667 napi_schedule(&channel->napi); in gsi_channel_trans_cancel_pending()
673 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte()
674 struct gsi_ring *tre_ring = &channel->tre_ring; in gsi_trans_read_byte()
678 trans_info = &channel->trans_info; in gsi_trans_read_byte()
682 return -EBUSY; in gsi_trans_read_byte()
686 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in gsi_trans_read_byte()
689 tre_ring->index++; in gsi_trans_read_byte()
698 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte_done()
700 gsi_trans_tre_release(&channel->trans_info, 1); in gsi_trans_read_byte_done()
706 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_init()
707 u32 tre_count = channel->tre_count; in gsi_channel_trans_init()
715 trans_info = &channel->trans_info; in gsi_channel_trans_init()
722 tre_max = gsi_channel_tre_max(channel->gsi, channel_id); in gsi_channel_trans_init()
723 atomic_set(&trans_info->tre_avail, tre_max); in gsi_channel_trans_init()
729 * power-of-two number of transactions we can use an index in gsi_channel_trans_init()
733 trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans), in gsi_channel_trans_init()
735 if (!trans_info->trans) in gsi_channel_trans_init()
736 return -ENOMEM; in gsi_channel_trans_init()
737 trans_info->free_id = 0; /* all modulo channel->tre_count */ in gsi_channel_trans_init()
738 trans_info->allocated_id = 0; in gsi_channel_trans_init()
739 trans_info->committed_id = 0; in gsi_channel_trans_init()
740 trans_info->pending_id = 0; in gsi_channel_trans_init()
741 trans_info->completed_id = 0; in gsi_channel_trans_init()
742 trans_info->polled_id = 0; in gsi_channel_trans_init()
749 trans_info->map = kcalloc(tre_count, sizeof(*trans_info->map), in gsi_channel_trans_init()
751 if (!trans_info->map) { in gsi_channel_trans_init()
752 ret = -ENOMEM; in gsi_channel_trans_init()
762 ret = gsi_trans_pool_init(&trans_info->sg_pool, in gsi_channel_trans_init()
764 tre_max, channel->trans_tre_max); in gsi_channel_trans_init()
772 kfree(trans_info->map); in gsi_channel_trans_init()
774 kfree(trans_info->trans); in gsi_channel_trans_init()
776 dev_err(gsi->dev, "error %d initializing channel %u transactions\n", in gsi_channel_trans_init()
785 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_exit()
787 gsi_trans_pool_exit(&trans_info->sg_pool); in gsi_channel_trans_exit()
788 kfree(trans_info->trans); in gsi_channel_trans_exit()
789 kfree(trans_info->map); in gsi_channel_trans_exit()