1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Xen grant DMA-mapping layer - contains special DMA-mapping routines 4 * for providing grant references as DMA addresses to be used by frontends 5 * (e.g. virtio) in Xen guests 6 * 7 * Copyright (c) 2021, Juergen Gross <jgross@suse.com> 8 */ 9 10 #include <linux/module.h> 11 #include <linux/dma-map-ops.h> 12 #include <linux/of.h> 13 #include <linux/pci.h> 14 #include <linux/pfn.h> 15 #include <linux/xarray.h> 16 #include <linux/virtio_anchor.h> 17 #include <linux/virtio.h> 18 #include <xen/xen.h> 19 #include <xen/xen-ops.h> 20 #include <xen/grant_table.h> 21 22 struct xen_grant_dma_data { 23 /* The ID of backend domain */ 24 domid_t backend_domid; 25 /* Is device behaving sane? */ 26 bool broken; 27 }; 28 29 static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ); 30 31 #define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63) 32 33 static inline dma_addr_t grant_to_dma(grant_ref_t grant) 34 { 35 return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT); 36 } 37 38 static inline grant_ref_t dma_to_grant(dma_addr_t dma) 39 { 40 return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT); 41 } 42 43 static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev) 44 { 45 struct xen_grant_dma_data *data; 46 unsigned long flags; 47 48 xa_lock_irqsave(&xen_grant_dma_devices, flags); 49 data = xa_load(&xen_grant_dma_devices, (unsigned long)dev); 50 xa_unlock_irqrestore(&xen_grant_dma_devices, flags); 51 52 return data; 53 } 54 55 static int store_xen_grant_dma_data(struct device *dev, 56 struct xen_grant_dma_data *data) 57 { 58 unsigned long flags; 59 int ret; 60 61 xa_lock_irqsave(&xen_grant_dma_devices, flags); 62 ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data, 63 GFP_ATOMIC)); 64 xa_unlock_irqrestore(&xen_grant_dma_devices, flags); 65 66 return ret; 67 } 68 69 /* 70 * DMA ops for Xen frontends (e.g. virtio). 71 * 72 * Used to act as a kind of software IOMMU for Xen guests by using grants as 73 * DMA addresses. 74 * Such a DMA address is formed by using the grant reference as a frame 75 * number and setting the highest address bit (this bit is for the backend 76 * to be able to distinguish it from e.g. a mmio address). 77 */ 78 static void *xen_grant_dma_alloc(struct device *dev, size_t size, 79 dma_addr_t *dma_handle, gfp_t gfp, 80 unsigned long attrs) 81 { 82 struct xen_grant_dma_data *data; 83 unsigned int i, n_pages = XEN_PFN_UP(size); 84 unsigned long pfn; 85 grant_ref_t grant; 86 void *ret; 87 88 data = find_xen_grant_dma_data(dev); 89 if (!data) 90 return NULL; 91 92 if (unlikely(data->broken)) 93 return NULL; 94 95 ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp); 96 if (!ret) 97 return NULL; 98 99 pfn = virt_to_pfn(ret); 100 101 if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) { 102 free_pages_exact(ret, n_pages * XEN_PAGE_SIZE); 103 return NULL; 104 } 105 106 for (i = 0; i < n_pages; i++) { 107 gnttab_grant_foreign_access_ref(grant + i, data->backend_domid, 108 pfn_to_gfn(pfn + i), 0); 109 } 110 111 *dma_handle = grant_to_dma(grant); 112 113 return ret; 114 } 115 116 static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr, 117 dma_addr_t dma_handle, unsigned long attrs) 118 { 119 struct xen_grant_dma_data *data; 120 unsigned int i, n_pages = XEN_PFN_UP(size); 121 grant_ref_t grant; 122 123 data = find_xen_grant_dma_data(dev); 124 if (!data) 125 return; 126 127 if (unlikely(data->broken)) 128 return; 129 130 grant = dma_to_grant(dma_handle); 131 132 for (i = 0; i < n_pages; i++) { 133 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { 134 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n"); 135 data->broken = true; 136 return; 137 } 138 } 139 140 gnttab_free_grant_reference_seq(grant, n_pages); 141 142 free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE); 143 } 144 145 static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size, 146 dma_addr_t *dma_handle, 147 enum dma_data_direction dir, 148 gfp_t gfp) 149 { 150 void *vaddr; 151 152 vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0); 153 if (!vaddr) 154 return NULL; 155 156 return virt_to_page(vaddr); 157 } 158 159 static void xen_grant_dma_free_pages(struct device *dev, size_t size, 160 struct page *vaddr, dma_addr_t dma_handle, 161 enum dma_data_direction dir) 162 { 163 xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0); 164 } 165 166 static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, 167 unsigned long offset, size_t size, 168 enum dma_data_direction dir, 169 unsigned long attrs) 170 { 171 struct xen_grant_dma_data *data; 172 unsigned long dma_offset = xen_offset_in_page(offset), 173 pfn_offset = XEN_PFN_DOWN(offset); 174 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size); 175 grant_ref_t grant; 176 dma_addr_t dma_handle; 177 178 if (WARN_ON(dir == DMA_NONE)) 179 return DMA_MAPPING_ERROR; 180 181 data = find_xen_grant_dma_data(dev); 182 if (!data) 183 return DMA_MAPPING_ERROR; 184 185 if (unlikely(data->broken)) 186 return DMA_MAPPING_ERROR; 187 188 if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) 189 return DMA_MAPPING_ERROR; 190 191 for (i = 0; i < n_pages; i++) { 192 gnttab_grant_foreign_access_ref(grant + i, data->backend_domid, 193 pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset), 194 dir == DMA_TO_DEVICE); 195 } 196 197 dma_handle = grant_to_dma(grant) + dma_offset; 198 199 return dma_handle; 200 } 201 202 static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 203 size_t size, enum dma_data_direction dir, 204 unsigned long attrs) 205 { 206 struct xen_grant_dma_data *data; 207 unsigned long dma_offset = xen_offset_in_page(dma_handle); 208 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size); 209 grant_ref_t grant; 210 211 if (WARN_ON(dir == DMA_NONE)) 212 return; 213 214 data = find_xen_grant_dma_data(dev); 215 if (!data) 216 return; 217 218 if (unlikely(data->broken)) 219 return; 220 221 grant = dma_to_grant(dma_handle); 222 223 for (i = 0; i < n_pages; i++) { 224 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { 225 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n"); 226 data->broken = true; 227 return; 228 } 229 } 230 231 gnttab_free_grant_reference_seq(grant, n_pages); 232 } 233 234 static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg, 235 int nents, enum dma_data_direction dir, 236 unsigned long attrs) 237 { 238 struct scatterlist *s; 239 unsigned int i; 240 241 if (WARN_ON(dir == DMA_NONE)) 242 return; 243 244 for_each_sg(sg, s, nents, i) 245 xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir, 246 attrs); 247 } 248 249 static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg, 250 int nents, enum dma_data_direction dir, 251 unsigned long attrs) 252 { 253 struct scatterlist *s; 254 unsigned int i; 255 256 if (WARN_ON(dir == DMA_NONE)) 257 return -EINVAL; 258 259 for_each_sg(sg, s, nents, i) { 260 s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset, 261 s->length, dir, attrs); 262 if (s->dma_address == DMA_MAPPING_ERROR) 263 goto out; 264 265 sg_dma_len(s) = s->length; 266 } 267 268 return nents; 269 270 out: 271 xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); 272 sg_dma_len(sg) = 0; 273 274 return -EIO; 275 } 276 277 static int xen_grant_dma_supported(struct device *dev, u64 mask) 278 { 279 return mask == DMA_BIT_MASK(64); 280 } 281 282 static const struct dma_map_ops xen_grant_dma_ops = { 283 .alloc = xen_grant_dma_alloc, 284 .free = xen_grant_dma_free, 285 .alloc_pages = xen_grant_dma_alloc_pages, 286 .free_pages = xen_grant_dma_free_pages, 287 .mmap = dma_common_mmap, 288 .get_sgtable = dma_common_get_sgtable, 289 .map_page = xen_grant_dma_map_page, 290 .unmap_page = xen_grant_dma_unmap_page, 291 .map_sg = xen_grant_dma_map_sg, 292 .unmap_sg = xen_grant_dma_unmap_sg, 293 .dma_supported = xen_grant_dma_supported, 294 }; 295 296 static struct device_node *xen_dt_get_node(struct device *dev) 297 { 298 if (dev_is_pci(dev)) { 299 struct pci_dev *pdev = to_pci_dev(dev); 300 struct pci_bus *bus = pdev->bus; 301 302 /* Walk up to the root bus to look for PCI Host controller */ 303 while (!pci_is_root_bus(bus)) 304 bus = bus->parent; 305 306 return of_node_get(bus->bridge->parent->of_node); 307 } 308 309 return of_node_get(dev->of_node); 310 } 311 312 static int xen_dt_grant_init_backend_domid(struct device *dev, 313 struct device_node *np, 314 domid_t *backend_domid) 315 { 316 struct of_phandle_args iommu_spec = { .args_count = 1 }; 317 318 if (dev_is_pci(dev)) { 319 struct pci_dev *pdev = to_pci_dev(dev); 320 u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn); 321 322 if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np, 323 iommu_spec.args)) { 324 dev_dbg(dev, "Cannot translate ID\n"); 325 return -ESRCH; 326 } 327 } else { 328 if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 329 0, &iommu_spec)) { 330 dev_dbg(dev, "Cannot parse iommus property\n"); 331 return -ESRCH; 332 } 333 } 334 335 if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") || 336 iommu_spec.args_count != 1) { 337 dev_dbg(dev, "Incompatible IOMMU node\n"); 338 of_node_put(iommu_spec.np); 339 return -ESRCH; 340 } 341 342 of_node_put(iommu_spec.np); 343 344 /* 345 * The endpoint ID here means the ID of the domain where the 346 * corresponding backend is running 347 */ 348 *backend_domid = iommu_spec.args[0]; 349 350 return 0; 351 } 352 353 static int xen_grant_init_backend_domid(struct device *dev, 354 domid_t *backend_domid) 355 { 356 struct device_node *np; 357 int ret = -ENODEV; 358 359 np = xen_dt_get_node(dev); 360 if (np) { 361 ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); 362 of_node_put(np); 363 } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { 364 dev_info(dev, "Using dom0 as backend\n"); 365 *backend_domid = 0; 366 ret = 0; 367 } 368 369 return ret; 370 } 371 372 static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid) 373 { 374 struct xen_grant_dma_data *data; 375 376 data = find_xen_grant_dma_data(dev); 377 if (data) { 378 dev_err(dev, "Xen grant DMA data is already created\n"); 379 return; 380 } 381 382 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 383 if (!data) 384 goto err; 385 386 data->backend_domid = backend_domid; 387 388 if (store_xen_grant_dma_data(dev, data)) { 389 dev_err(dev, "Cannot store Xen grant DMA data\n"); 390 goto err; 391 } 392 393 dev->dma_ops = &xen_grant_dma_ops; 394 395 return; 396 397 err: 398 devm_kfree(dev, data); 399 dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n"); 400 } 401 402 bool xen_virtio_restricted_mem_acc(struct virtio_device *dev) 403 { 404 domid_t backend_domid; 405 406 if (!xen_grant_init_backend_domid(dev->dev.parent, &backend_domid)) { 407 xen_grant_setup_dma_ops(dev->dev.parent, backend_domid); 408 return true; 409 } 410 411 return false; 412 } 413 414 MODULE_DESCRIPTION("Xen grant DMA-mapping layer"); 415 MODULE_AUTHOR("Juergen Gross <jgross@suse.com>"); 416 MODULE_LICENSE("GPL"); 417