xref: /openbmc/linux/drivers/xen/grant-dma-ops.c (revision 9a29f5fc) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Xen grant DMA-mapping layer - contains special DMA-mapping routines
4  * for providing grant references as DMA addresses to be used by frontends
5  * (e.g. virtio) in Xen guests
6  *
7  * Copyright (c) 2021, Juergen Gross <jgross@suse.com>
8  */
9 
10 #include <linux/module.h>
11 #include <linux/dma-map-ops.h>
12 #include <linux/of.h>
13 #include <linux/pfn.h>
14 #include <linux/xarray.h>
15 #include <linux/virtio_anchor.h>
16 #include <linux/virtio.h>
17 #include <xen/xen.h>
18 #include <xen/xen-ops.h>
19 #include <xen/grant_table.h>
20 
21 struct xen_grant_dma_data {
22 	/* The ID of backend domain */
23 	domid_t backend_domid;
24 	/* Is device behaving sane? */
25 	bool broken;
26 };
27 
28 static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
29 
30 #define XEN_GRANT_DMA_ADDR_OFF	(1ULL << 63)
31 
32 static inline dma_addr_t grant_to_dma(grant_ref_t grant)
33 {
34 	return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);
35 }
36 
37 static inline grant_ref_t dma_to_grant(dma_addr_t dma)
38 {
39 	return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);
40 }
41 
42 static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
43 {
44 	struct xen_grant_dma_data *data;
45 	unsigned long flags;
46 
47 	xa_lock_irqsave(&xen_grant_dma_devices, flags);
48 	data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);
49 	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
50 
51 	return data;
52 }
53 
54 static int store_xen_grant_dma_data(struct device *dev,
55 				    struct xen_grant_dma_data *data)
56 {
57 	unsigned long flags;
58 	int ret;
59 
60 	xa_lock_irqsave(&xen_grant_dma_devices, flags);
61 	ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
62 			GFP_ATOMIC));
63 	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
64 
65 	return ret;
66 }
67 
68 /*
69  * DMA ops for Xen frontends (e.g. virtio).
70  *
71  * Used to act as a kind of software IOMMU for Xen guests by using grants as
72  * DMA addresses.
73  * Such a DMA address is formed by using the grant reference as a frame
74  * number and setting the highest address bit (this bit is for the backend
75  * to be able to distinguish it from e.g. a mmio address).
76  */
77 static void *xen_grant_dma_alloc(struct device *dev, size_t size,
78 				 dma_addr_t *dma_handle, gfp_t gfp,
79 				 unsigned long attrs)
80 {
81 	struct xen_grant_dma_data *data;
82 	unsigned int i, n_pages = XEN_PFN_UP(size);
83 	unsigned long pfn;
84 	grant_ref_t grant;
85 	void *ret;
86 
87 	data = find_xen_grant_dma_data(dev);
88 	if (!data)
89 		return NULL;
90 
91 	if (unlikely(data->broken))
92 		return NULL;
93 
94 	ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp);
95 	if (!ret)
96 		return NULL;
97 
98 	pfn = virt_to_pfn(ret);
99 
100 	if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) {
101 		free_pages_exact(ret, n_pages * XEN_PAGE_SIZE);
102 		return NULL;
103 	}
104 
105 	for (i = 0; i < n_pages; i++) {
106 		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
107 				pfn_to_gfn(pfn + i), 0);
108 	}
109 
110 	*dma_handle = grant_to_dma(grant);
111 
112 	return ret;
113 }
114 
115 static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,
116 			       dma_addr_t dma_handle, unsigned long attrs)
117 {
118 	struct xen_grant_dma_data *data;
119 	unsigned int i, n_pages = XEN_PFN_UP(size);
120 	grant_ref_t grant;
121 
122 	data = find_xen_grant_dma_data(dev);
123 	if (!data)
124 		return;
125 
126 	if (unlikely(data->broken))
127 		return;
128 
129 	grant = dma_to_grant(dma_handle);
130 
131 	for (i = 0; i < n_pages; i++) {
132 		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
133 			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
134 			data->broken = true;
135 			return;
136 		}
137 	}
138 
139 	gnttab_free_grant_reference_seq(grant, n_pages);
140 
141 	free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE);
142 }
143 
144 static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,
145 					      dma_addr_t *dma_handle,
146 					      enum dma_data_direction dir,
147 					      gfp_t gfp)
148 {
149 	void *vaddr;
150 
151 	vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0);
152 	if (!vaddr)
153 		return NULL;
154 
155 	return virt_to_page(vaddr);
156 }
157 
158 static void xen_grant_dma_free_pages(struct device *dev, size_t size,
159 				     struct page *vaddr, dma_addr_t dma_handle,
160 				     enum dma_data_direction dir)
161 {
162 	xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0);
163 }
164 
165 static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page,
166 					 unsigned long offset, size_t size,
167 					 enum dma_data_direction dir,
168 					 unsigned long attrs)
169 {
170 	struct xen_grant_dma_data *data;
171 	unsigned long dma_offset = xen_offset_in_page(offset),
172 			pfn_offset = XEN_PFN_DOWN(offset);
173 	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
174 	grant_ref_t grant;
175 	dma_addr_t dma_handle;
176 
177 	if (WARN_ON(dir == DMA_NONE))
178 		return DMA_MAPPING_ERROR;
179 
180 	data = find_xen_grant_dma_data(dev);
181 	if (!data)
182 		return DMA_MAPPING_ERROR;
183 
184 	if (unlikely(data->broken))
185 		return DMA_MAPPING_ERROR;
186 
187 	if (gnttab_alloc_grant_reference_seq(n_pages, &grant))
188 		return DMA_MAPPING_ERROR;
189 
190 	for (i = 0; i < n_pages; i++) {
191 		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
192 				pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset),
193 				dir == DMA_TO_DEVICE);
194 	}
195 
196 	dma_handle = grant_to_dma(grant) + dma_offset;
197 
198 	return dma_handle;
199 }
200 
201 static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
202 				     size_t size, enum dma_data_direction dir,
203 				     unsigned long attrs)
204 {
205 	struct xen_grant_dma_data *data;
206 	unsigned long dma_offset = xen_offset_in_page(dma_handle);
207 	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
208 	grant_ref_t grant;
209 
210 	if (WARN_ON(dir == DMA_NONE))
211 		return;
212 
213 	data = find_xen_grant_dma_data(dev);
214 	if (!data)
215 		return;
216 
217 	if (unlikely(data->broken))
218 		return;
219 
220 	grant = dma_to_grant(dma_handle);
221 
222 	for (i = 0; i < n_pages; i++) {
223 		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
224 			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
225 			data->broken = true;
226 			return;
227 		}
228 	}
229 
230 	gnttab_free_grant_reference_seq(grant, n_pages);
231 }
232 
233 static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
234 				   int nents, enum dma_data_direction dir,
235 				   unsigned long attrs)
236 {
237 	struct scatterlist *s;
238 	unsigned int i;
239 
240 	if (WARN_ON(dir == DMA_NONE))
241 		return;
242 
243 	for_each_sg(sg, s, nents, i)
244 		xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir,
245 				attrs);
246 }
247 
248 static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,
249 				int nents, enum dma_data_direction dir,
250 				unsigned long attrs)
251 {
252 	struct scatterlist *s;
253 	unsigned int i;
254 
255 	if (WARN_ON(dir == DMA_NONE))
256 		return -EINVAL;
257 
258 	for_each_sg(sg, s, nents, i) {
259 		s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset,
260 				s->length, dir, attrs);
261 		if (s->dma_address == DMA_MAPPING_ERROR)
262 			goto out;
263 
264 		sg_dma_len(s) = s->length;
265 	}
266 
267 	return nents;
268 
269 out:
270 	xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
271 	sg_dma_len(sg) = 0;
272 
273 	return -EIO;
274 }
275 
276 static int xen_grant_dma_supported(struct device *dev, u64 mask)
277 {
278 	return mask == DMA_BIT_MASK(64);
279 }
280 
281 static const struct dma_map_ops xen_grant_dma_ops = {
282 	.alloc = xen_grant_dma_alloc,
283 	.free = xen_grant_dma_free,
284 	.alloc_pages = xen_grant_dma_alloc_pages,
285 	.free_pages = xen_grant_dma_free_pages,
286 	.mmap = dma_common_mmap,
287 	.get_sgtable = dma_common_get_sgtable,
288 	.map_page = xen_grant_dma_map_page,
289 	.unmap_page = xen_grant_dma_unmap_page,
290 	.map_sg = xen_grant_dma_map_sg,
291 	.unmap_sg = xen_grant_dma_unmap_sg,
292 	.dma_supported = xen_grant_dma_supported,
293 };
294 
295 static bool xen_is_dt_grant_dma_device(struct device *dev)
296 {
297 	struct device_node *iommu_np;
298 	bool has_iommu;
299 
300 	iommu_np = of_parse_phandle(dev->of_node, "iommus", 0);
301 	has_iommu = iommu_np &&
302 		    of_device_is_compatible(iommu_np, "xen,grant-dma");
303 	of_node_put(iommu_np);
304 
305 	return has_iommu;
306 }
307 
308 bool xen_is_grant_dma_device(struct device *dev)
309 {
310 	/* XXX Handle only DT devices for now */
311 	if (dev->of_node)
312 		return xen_is_dt_grant_dma_device(dev);
313 
314 	return false;
315 }
316 
317 bool xen_virtio_mem_acc(struct virtio_device *dev)
318 {
319 	if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())
320 		return true;
321 
322 	return xen_is_grant_dma_device(dev->dev.parent);
323 }
324 
325 static int xen_dt_grant_init_backend_domid(struct device *dev,
326 					   struct xen_grant_dma_data *data)
327 {
328 	struct of_phandle_args iommu_spec;
329 
330 	if (of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells",
331 			0, &iommu_spec)) {
332 		dev_err(dev, "Cannot parse iommus property\n");
333 		return -ESRCH;
334 	}
335 
336 	if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||
337 			iommu_spec.args_count != 1) {
338 		dev_err(dev, "Incompatible IOMMU node\n");
339 		of_node_put(iommu_spec.np);
340 		return -ESRCH;
341 	}
342 
343 	of_node_put(iommu_spec.np);
344 
345 	/*
346 	 * The endpoint ID here means the ID of the domain where the
347 	 * corresponding backend is running
348 	 */
349 	data->backend_domid = iommu_spec.args[0];
350 
351 	return 0;
352 }
353 
354 void xen_grant_setup_dma_ops(struct device *dev)
355 {
356 	struct xen_grant_dma_data *data;
357 
358 	data = find_xen_grant_dma_data(dev);
359 	if (data) {
360 		dev_err(dev, "Xen grant DMA data is already created\n");
361 		return;
362 	}
363 
364 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
365 	if (!data)
366 		goto err;
367 
368 	if (dev->of_node) {
369 		if (xen_dt_grant_init_backend_domid(dev, data))
370 			goto err;
371 	} else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT)) {
372 		dev_info(dev, "Using dom0 as backend\n");
373 		data->backend_domid = 0;
374 	} else {
375 		/* XXX ACPI device unsupported for now */
376 		goto err;
377 	}
378 
379 	if (store_xen_grant_dma_data(dev, data)) {
380 		dev_err(dev, "Cannot store Xen grant DMA data\n");
381 		goto err;
382 	}
383 
384 	dev->dma_ops = &xen_grant_dma_ops;
385 
386 	return;
387 
388 err:
389 	devm_kfree(dev, data);
390 	dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
391 }
392 
393 bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
394 {
395 	bool ret = xen_virtio_mem_acc(dev);
396 
397 	if (ret)
398 		xen_grant_setup_dma_ops(dev->dev.parent);
399 
400 	return ret;
401 }
402 
403 MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
404 MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
405 MODULE_LICENSE("GPL");
406