xref: /openbmc/linux/drivers/xen/grant-dma-ops.c (revision c2a311dc) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Xen grant DMA-mapping layer - contains special DMA-mapping routines
4  * for providing grant references as DMA addresses to be used by frontends
5  * (e.g. virtio) in Xen guests
6  *
7  * Copyright (c) 2021, Juergen Gross <jgross@suse.com>
8  */
9 
10 #include <linux/module.h>
11 #include <linux/dma-map-ops.h>
12 #include <linux/of.h>
13 #include <linux/pci.h>
14 #include <linux/pfn.h>
15 #include <linux/xarray.h>
16 #include <linux/virtio_anchor.h>
17 #include <linux/virtio.h>
18 #include <xen/xen.h>
19 #include <xen/xen-ops.h>
20 #include <xen/grant_table.h>
21 
22 struct xen_grant_dma_data {
23 	/* The ID of backend domain */
24 	domid_t backend_domid;
25 	/* Is device behaving sane? */
26 	bool broken;
27 };
28 
29 static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
30 
31 #define XEN_GRANT_DMA_ADDR_OFF	(1ULL << 63)
32 
33 static inline dma_addr_t grant_to_dma(grant_ref_t grant)
34 {
35 	return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);
36 }
37 
38 static inline grant_ref_t dma_to_grant(dma_addr_t dma)
39 {
40 	return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);
41 }
42 
43 static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
44 {
45 	struct xen_grant_dma_data *data;
46 	unsigned long flags;
47 
48 	xa_lock_irqsave(&xen_grant_dma_devices, flags);
49 	data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);
50 	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
51 
52 	return data;
53 }
54 
55 static int store_xen_grant_dma_data(struct device *dev,
56 				    struct xen_grant_dma_data *data)
57 {
58 	unsigned long flags;
59 	int ret;
60 
61 	xa_lock_irqsave(&xen_grant_dma_devices, flags);
62 	ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
63 			GFP_ATOMIC));
64 	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
65 
66 	return ret;
67 }
68 
69 /*
70  * DMA ops for Xen frontends (e.g. virtio).
71  *
72  * Used to act as a kind of software IOMMU for Xen guests by using grants as
73  * DMA addresses.
74  * Such a DMA address is formed by using the grant reference as a frame
75  * number and setting the highest address bit (this bit is for the backend
76  * to be able to distinguish it from e.g. a mmio address).
77  */
78 static void *xen_grant_dma_alloc(struct device *dev, size_t size,
79 				 dma_addr_t *dma_handle, gfp_t gfp,
80 				 unsigned long attrs)
81 {
82 	struct xen_grant_dma_data *data;
83 	unsigned int i, n_pages = XEN_PFN_UP(size);
84 	unsigned long pfn;
85 	grant_ref_t grant;
86 	void *ret;
87 
88 	data = find_xen_grant_dma_data(dev);
89 	if (!data)
90 		return NULL;
91 
92 	if (unlikely(data->broken))
93 		return NULL;
94 
95 	ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp);
96 	if (!ret)
97 		return NULL;
98 
99 	pfn = virt_to_pfn(ret);
100 
101 	if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) {
102 		free_pages_exact(ret, n_pages * XEN_PAGE_SIZE);
103 		return NULL;
104 	}
105 
106 	for (i = 0; i < n_pages; i++) {
107 		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
108 				pfn_to_gfn(pfn + i), 0);
109 	}
110 
111 	*dma_handle = grant_to_dma(grant);
112 
113 	return ret;
114 }
115 
116 static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,
117 			       dma_addr_t dma_handle, unsigned long attrs)
118 {
119 	struct xen_grant_dma_data *data;
120 	unsigned int i, n_pages = XEN_PFN_UP(size);
121 	grant_ref_t grant;
122 
123 	data = find_xen_grant_dma_data(dev);
124 	if (!data)
125 		return;
126 
127 	if (unlikely(data->broken))
128 		return;
129 
130 	grant = dma_to_grant(dma_handle);
131 
132 	for (i = 0; i < n_pages; i++) {
133 		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
134 			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
135 			data->broken = true;
136 			return;
137 		}
138 	}
139 
140 	gnttab_free_grant_reference_seq(grant, n_pages);
141 
142 	free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE);
143 }
144 
145 static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,
146 					      dma_addr_t *dma_handle,
147 					      enum dma_data_direction dir,
148 					      gfp_t gfp)
149 {
150 	void *vaddr;
151 
152 	vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0);
153 	if (!vaddr)
154 		return NULL;
155 
156 	return virt_to_page(vaddr);
157 }
158 
159 static void xen_grant_dma_free_pages(struct device *dev, size_t size,
160 				     struct page *vaddr, dma_addr_t dma_handle,
161 				     enum dma_data_direction dir)
162 {
163 	xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0);
164 }
165 
166 static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page,
167 					 unsigned long offset, size_t size,
168 					 enum dma_data_direction dir,
169 					 unsigned long attrs)
170 {
171 	struct xen_grant_dma_data *data;
172 	unsigned long dma_offset = xen_offset_in_page(offset),
173 			pfn_offset = XEN_PFN_DOWN(offset);
174 	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
175 	grant_ref_t grant;
176 	dma_addr_t dma_handle;
177 
178 	if (WARN_ON(dir == DMA_NONE))
179 		return DMA_MAPPING_ERROR;
180 
181 	data = find_xen_grant_dma_data(dev);
182 	if (!data)
183 		return DMA_MAPPING_ERROR;
184 
185 	if (unlikely(data->broken))
186 		return DMA_MAPPING_ERROR;
187 
188 	if (gnttab_alloc_grant_reference_seq(n_pages, &grant))
189 		return DMA_MAPPING_ERROR;
190 
191 	for (i = 0; i < n_pages; i++) {
192 		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
193 				pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset),
194 				dir == DMA_TO_DEVICE);
195 	}
196 
197 	dma_handle = grant_to_dma(grant) + dma_offset;
198 
199 	return dma_handle;
200 }
201 
202 static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
203 				     size_t size, enum dma_data_direction dir,
204 				     unsigned long attrs)
205 {
206 	struct xen_grant_dma_data *data;
207 	unsigned long dma_offset = xen_offset_in_page(dma_handle);
208 	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
209 	grant_ref_t grant;
210 
211 	if (WARN_ON(dir == DMA_NONE))
212 		return;
213 
214 	data = find_xen_grant_dma_data(dev);
215 	if (!data)
216 		return;
217 
218 	if (unlikely(data->broken))
219 		return;
220 
221 	grant = dma_to_grant(dma_handle);
222 
223 	for (i = 0; i < n_pages; i++) {
224 		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
225 			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
226 			data->broken = true;
227 			return;
228 		}
229 	}
230 
231 	gnttab_free_grant_reference_seq(grant, n_pages);
232 }
233 
234 static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
235 				   int nents, enum dma_data_direction dir,
236 				   unsigned long attrs)
237 {
238 	struct scatterlist *s;
239 	unsigned int i;
240 
241 	if (WARN_ON(dir == DMA_NONE))
242 		return;
243 
244 	for_each_sg(sg, s, nents, i)
245 		xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir,
246 				attrs);
247 }
248 
249 static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,
250 				int nents, enum dma_data_direction dir,
251 				unsigned long attrs)
252 {
253 	struct scatterlist *s;
254 	unsigned int i;
255 
256 	if (WARN_ON(dir == DMA_NONE))
257 		return -EINVAL;
258 
259 	for_each_sg(sg, s, nents, i) {
260 		s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset,
261 				s->length, dir, attrs);
262 		if (s->dma_address == DMA_MAPPING_ERROR)
263 			goto out;
264 
265 		sg_dma_len(s) = s->length;
266 	}
267 
268 	return nents;
269 
270 out:
271 	xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
272 	sg_dma_len(sg) = 0;
273 
274 	return -EIO;
275 }
276 
277 static int xen_grant_dma_supported(struct device *dev, u64 mask)
278 {
279 	return mask == DMA_BIT_MASK(64);
280 }
281 
282 static const struct dma_map_ops xen_grant_dma_ops = {
283 	.alloc = xen_grant_dma_alloc,
284 	.free = xen_grant_dma_free,
285 	.alloc_pages = xen_grant_dma_alloc_pages,
286 	.free_pages = xen_grant_dma_free_pages,
287 	.mmap = dma_common_mmap,
288 	.get_sgtable = dma_common_get_sgtable,
289 	.map_page = xen_grant_dma_map_page,
290 	.unmap_page = xen_grant_dma_unmap_page,
291 	.map_sg = xen_grant_dma_map_sg,
292 	.unmap_sg = xen_grant_dma_unmap_sg,
293 	.dma_supported = xen_grant_dma_supported,
294 };
295 
296 static struct device_node *xen_dt_get_node(struct device *dev)
297 {
298 	if (dev_is_pci(dev)) {
299 		struct pci_dev *pdev = to_pci_dev(dev);
300 		struct pci_bus *bus = pdev->bus;
301 
302 		/* Walk up to the root bus to look for PCI Host controller */
303 		while (!pci_is_root_bus(bus))
304 			bus = bus->parent;
305 
306 		if (!bus->bridge->parent)
307 			return NULL;
308 		return of_node_get(bus->bridge->parent->of_node);
309 	}
310 
311 	return of_node_get(dev->of_node);
312 }
313 
314 static int xen_dt_grant_init_backend_domid(struct device *dev,
315 					   struct device_node *np,
316 					   domid_t *backend_domid)
317 {
318 	struct of_phandle_args iommu_spec = { .args_count = 1 };
319 
320 	if (dev_is_pci(dev)) {
321 		struct pci_dev *pdev = to_pci_dev(dev);
322 		u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
323 
324 		if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np,
325 				iommu_spec.args)) {
326 			dev_dbg(dev, "Cannot translate ID\n");
327 			return -ESRCH;
328 		}
329 	} else {
330 		if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
331 				0, &iommu_spec)) {
332 			dev_dbg(dev, "Cannot parse iommus property\n");
333 			return -ESRCH;
334 		}
335 	}
336 
337 	if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||
338 			iommu_spec.args_count != 1) {
339 		dev_dbg(dev, "Incompatible IOMMU node\n");
340 		of_node_put(iommu_spec.np);
341 		return -ESRCH;
342 	}
343 
344 	of_node_put(iommu_spec.np);
345 
346 	/*
347 	 * The endpoint ID here means the ID of the domain where the
348 	 * corresponding backend is running
349 	 */
350 	*backend_domid = iommu_spec.args[0];
351 
352 	return 0;
353 }
354 
355 static int xen_grant_init_backend_domid(struct device *dev,
356 					domid_t *backend_domid)
357 {
358 	struct device_node *np;
359 	int ret = -ENODEV;
360 
361 	np = xen_dt_get_node(dev);
362 	if (np) {
363 		ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid);
364 		of_node_put(np);
365 	} else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) {
366 		dev_info(dev, "Using dom0 as backend\n");
367 		*backend_domid = 0;
368 		ret = 0;
369 	}
370 
371 	return ret;
372 }
373 
374 static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid)
375 {
376 	struct xen_grant_dma_data *data;
377 
378 	data = find_xen_grant_dma_data(dev);
379 	if (data) {
380 		dev_err(dev, "Xen grant DMA data is already created\n");
381 		return;
382 	}
383 
384 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
385 	if (!data)
386 		goto err;
387 
388 	data->backend_domid = backend_domid;
389 
390 	if (store_xen_grant_dma_data(dev, data)) {
391 		dev_err(dev, "Cannot store Xen grant DMA data\n");
392 		goto err;
393 	}
394 
395 	dev->dma_ops = &xen_grant_dma_ops;
396 
397 	return;
398 
399 err:
400 	devm_kfree(dev, data);
401 	dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
402 }
403 
404 bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
405 {
406 	domid_t backend_domid;
407 
408 	if (!xen_grant_init_backend_domid(dev->dev.parent, &backend_domid)) {
409 		xen_grant_setup_dma_ops(dev->dev.parent, backend_domid);
410 		return true;
411 	}
412 
413 	return false;
414 }
415 
416 MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
417 MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
418 MODULE_LICENSE("GPL");
419