xref: /openbmc/linux/include/linux/swiotlb.h (revision 2d5780bb)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_SWIOTLB_H
3 #define __LINUX_SWIOTLB_H
4 
5 #include <linux/device.h>
6 #include <linux/dma-direction.h>
7 #include <linux/init.h>
8 #include <linux/types.h>
9 #include <linux/limits.h>
10 #include <linux/spinlock.h>
11 #include <linux/workqueue.h>
12 
13 struct device;
14 struct page;
15 struct scatterlist;
16 
17 #define SWIOTLB_VERBOSE	(1 << 0) /* verbose initialization */
18 #define SWIOTLB_FORCE	(1 << 1) /* force bounce buffering */
19 #define SWIOTLB_ANY	(1 << 2) /* allow any memory for the buffer */
20 
21 /*
22  * Maximum allowable number of contiguous slabs to map,
23  * must be a power of 2.  What is the appropriate value ?
24  * The complexity of {map,unmap}_single is linearly dependent on this value.
25  */
26 #define IO_TLB_SEGSIZE	128
27 
28 /*
29  * log of the size of each IO TLB slab.  The number of slabs is command line
30  * controllable.
31  */
32 #define IO_TLB_SHIFT 11
33 #define IO_TLB_SIZE (1 << IO_TLB_SHIFT)
34 
35 /* default to 64MB */
36 #define IO_TLB_DEFAULT_SIZE (64UL<<20)
37 
38 unsigned long swiotlb_size_or_default(void);
39 void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
40 	int (*remap)(void *tlb, unsigned long nslabs));
41 int swiotlb_init_late(size_t size, gfp_t gfp_mask,
42 	int (*remap)(void *tlb, unsigned long nslabs));
43 extern void __init swiotlb_update_mem_attributes(void);
44 
45 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
46 		size_t mapping_size, size_t alloc_size,
47 		unsigned int alloc_aligned_mask, enum dma_data_direction dir,
48 		unsigned long attrs);
49 
50 extern void swiotlb_tbl_unmap_single(struct device *hwdev,
51 				     phys_addr_t tlb_addr,
52 				     size_t mapping_size,
53 				     enum dma_data_direction dir,
54 				     unsigned long attrs);
55 
56 void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
57 		size_t size, enum dma_data_direction dir);
58 void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
59 		size_t size, enum dma_data_direction dir);
60 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
61 		size_t size, enum dma_data_direction dir, unsigned long attrs);
62 
63 #ifdef CONFIG_SWIOTLB
64 
65 /**
66  * struct io_tlb_pool - IO TLB memory pool descriptor
67  * @start:	The start address of the swiotlb memory pool. Used to do a quick
68  *		range check to see if the memory was in fact allocated by this
69  *		API.
70  * @end:	The end address of the swiotlb memory pool. Used to do a quick
71  *		range check to see if the memory was in fact allocated by this
72  *		API.
73  * @vaddr:	The vaddr of the swiotlb memory pool. The swiotlb memory pool
74  *		may be remapped in the memory encrypted case and store virtual
75  *		address for bounce buffer operation.
76  * @nslabs:	The number of IO TLB slots between @start and @end. For the
77  *		default swiotlb, this can be adjusted with a boot parameter,
78  *		see setup_io_tlb_npages().
79  * @late_alloc:	%true if allocated using the page allocator.
80  * @nareas:	Number of areas in the pool.
81  * @area_nslabs: Number of slots in each area.
82  * @areas:	Array of memory area descriptors.
83  * @slots:	Array of slot descriptors.
84  * @node:	Member of the IO TLB memory pool list.
85  * @rcu:	RCU head for swiotlb_dyn_free().
86  * @transient:  %true if transient memory pool.
87  */
88 struct io_tlb_pool {
89 	phys_addr_t start;
90 	phys_addr_t end;
91 	void *vaddr;
92 	unsigned long nslabs;
93 	bool late_alloc;
94 	unsigned int nareas;
95 	unsigned int area_nslabs;
96 	struct io_tlb_area *areas;
97 	struct io_tlb_slot *slots;
98 #ifdef CONFIG_SWIOTLB_DYNAMIC
99 	struct list_head node;
100 	struct rcu_head rcu;
101 	bool transient;
102 #endif
103 };
104 
105 /**
106  * struct io_tlb_mem - Software IO TLB allocator
107  * @defpool:	Default (initial) IO TLB memory pool descriptor.
108  * @pool:	IO TLB memory pool descriptor (if not dynamic).
109  * @nslabs:	Total number of IO TLB slabs in all pools.
110  * @debugfs:	The dentry to debugfs.
111  * @force_bounce: %true if swiotlb bouncing is forced
112  * @for_alloc:  %true if the pool is used for memory allocation
113  * @can_grow:	%true if more pools can be allocated dynamically.
114  * @phys_limit:	Maximum allowed physical address.
115  * @lock:	Lock to synchronize changes to the list.
116  * @pools:	List of IO TLB memory pool descriptors (if dynamic).
117  * @dyn_alloc:	Dynamic IO TLB pool allocation work.
118  * @total_used:	The total number of slots in the pool that are currently used
119  *		across all areas. Used only for calculating used_hiwater in
120  *		debugfs.
121  * @used_hiwater: The high water mark for total_used.  Used only for reporting
122  *		in debugfs.
123  */
124 struct io_tlb_mem {
125 	struct io_tlb_pool defpool;
126 	unsigned long nslabs;
127 	struct dentry *debugfs;
128 	bool force_bounce;
129 	bool for_alloc;
130 #ifdef CONFIG_SWIOTLB_DYNAMIC
131 	bool can_grow;
132 	u64 phys_limit;
133 	spinlock_t lock;
134 	struct list_head pools;
135 	struct work_struct dyn_alloc;
136 #endif
137 #ifdef CONFIG_DEBUG_FS
138 	atomic_long_t total_used;
139 	atomic_long_t used_hiwater;
140 #endif
141 };
142 
143 #ifdef CONFIG_SWIOTLB_DYNAMIC
144 
145 struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
146 
147 #else
148 
swiotlb_find_pool(struct device * dev,phys_addr_t paddr)149 static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
150 						    phys_addr_t paddr)
151 {
152 	return &dev->dma_io_tlb_mem->defpool;
153 }
154 
155 #endif
156 
157 /**
158  * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb
159  * @dev:        Device which has mapped the buffer.
160  * @paddr:      Physical address within the DMA buffer.
161  *
162  * Check if @paddr points into a bounce buffer.
163  *
164  * Return:
165  * * %true if @paddr points into a bounce buffer
166  * * %false otherwise
167  */
is_swiotlb_buffer(struct device * dev,phys_addr_t paddr)168 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
169 {
170 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
171 
172 	if (!mem)
173 		return false;
174 
175 #ifdef CONFIG_SWIOTLB_DYNAMIC
176 	/*
177 	 * All SWIOTLB buffer addresses must have been returned by
178 	 * swiotlb_tbl_map_single() and passed to a device driver.
179 	 * If a SWIOTLB address is checked on another CPU, then it was
180 	 * presumably loaded by the device driver from an unspecified private
181 	 * data structure. Make sure that this load is ordered before reading
182 	 * dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().
183 	 *
184 	 * This barrier pairs with smp_mb() in swiotlb_find_slots().
185 	 */
186 	smp_rmb();
187 	return READ_ONCE(dev->dma_uses_io_tlb) &&
188 		swiotlb_find_pool(dev, paddr);
189 #else
190 	return paddr >= mem->defpool.start && paddr < mem->defpool.end;
191 #endif
192 }
193 
is_swiotlb_force_bounce(struct device * dev)194 static inline bool is_swiotlb_force_bounce(struct device *dev)
195 {
196 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
197 
198 	return mem && mem->force_bounce;
199 }
200 
201 void swiotlb_init(bool addressing_limited, unsigned int flags);
202 void __init swiotlb_exit(void);
203 void swiotlb_dev_init(struct device *dev);
204 size_t swiotlb_max_mapping_size(struct device *dev);
205 bool is_swiotlb_allocated(void);
206 bool is_swiotlb_active(struct device *dev);
207 void __init swiotlb_adjust_size(unsigned long size);
208 phys_addr_t default_swiotlb_base(void);
209 phys_addr_t default_swiotlb_limit(void);
210 #else
swiotlb_init(bool addressing_limited,unsigned int flags)211 static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
212 {
213 }
214 
swiotlb_dev_init(struct device * dev)215 static inline void swiotlb_dev_init(struct device *dev)
216 {
217 }
218 
is_swiotlb_buffer(struct device * dev,phys_addr_t paddr)219 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
220 {
221 	return false;
222 }
is_swiotlb_force_bounce(struct device * dev)223 static inline bool is_swiotlb_force_bounce(struct device *dev)
224 {
225 	return false;
226 }
swiotlb_exit(void)227 static inline void swiotlb_exit(void)
228 {
229 }
swiotlb_max_mapping_size(struct device * dev)230 static inline size_t swiotlb_max_mapping_size(struct device *dev)
231 {
232 	return SIZE_MAX;
233 }
234 
is_swiotlb_allocated(void)235 static inline bool is_swiotlb_allocated(void)
236 {
237 	return false;
238 }
239 
is_swiotlb_active(struct device * dev)240 static inline bool is_swiotlb_active(struct device *dev)
241 {
242 	return false;
243 }
244 
swiotlb_adjust_size(unsigned long size)245 static inline void swiotlb_adjust_size(unsigned long size)
246 {
247 }
248 
default_swiotlb_base(void)249 static inline phys_addr_t default_swiotlb_base(void)
250 {
251 	return 0;
252 }
253 
default_swiotlb_limit(void)254 static inline phys_addr_t default_swiotlb_limit(void)
255 {
256 	return 0;
257 }
258 #endif /* CONFIG_SWIOTLB */
259 
260 extern void swiotlb_print_info(void);
261 
262 #ifdef CONFIG_DMA_RESTRICTED_POOL
263 struct page *swiotlb_alloc(struct device *dev, size_t size);
264 bool swiotlb_free(struct device *dev, struct page *page, size_t size);
265 
is_swiotlb_for_alloc(struct device * dev)266 static inline bool is_swiotlb_for_alloc(struct device *dev)
267 {
268 	return dev->dma_io_tlb_mem->for_alloc;
269 }
270 #else
swiotlb_alloc(struct device * dev,size_t size)271 static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
272 {
273 	return NULL;
274 }
swiotlb_free(struct device * dev,struct page * page,size_t size)275 static inline bool swiotlb_free(struct device *dev, struct page *page,
276 				size_t size)
277 {
278 	return false;
279 }
is_swiotlb_for_alloc(struct device * dev)280 static inline bool is_swiotlb_for_alloc(struct device *dev)
281 {
282 	return false;
283 }
284 #endif /* CONFIG_DMA_RESTRICTED_POOL */
285 
286 #endif /* __LINUX_SWIOTLB_H */
287