1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/cpu.h> 3 #include <linux/dma-direct.h> 4 #include <linux/dma-map-ops.h> 5 #include <linux/gfp.h> 6 #include <linux/highmem.h> 7 #include <linux/export.h> 8 #include <linux/memblock.h> 9 #include <linux/of_address.h> 10 #include <linux/slab.h> 11 #include <linux/types.h> 12 #include <linux/vmalloc.h> 13 #include <linux/swiotlb.h> 14 15 #include <xen/xen.h> 16 #include <xen/interface/grant_table.h> 17 #include <xen/interface/memory.h> 18 #include <xen/page.h> 19 #include <xen/xen-ops.h> 20 #include <xen/swiotlb-xen.h> 21 22 #include <asm/cacheflush.h> 23 #include <asm/xen/hypercall.h> 24 #include <asm/xen/interface.h> 25 xen_swiotlb_gfp(void)26 static gfp_t xen_swiotlb_gfp(void) 27 { 28 phys_addr_t base; 29 u64 i; 30 31 for_each_mem_range(i, &base, NULL) { 32 if (base < (phys_addr_t)0xffffffff) { 33 if (IS_ENABLED(CONFIG_ZONE_DMA32)) 34 return __GFP_DMA32; 35 return __GFP_DMA; 36 } 37 } 38 39 return GFP_KERNEL; 40 } 41 42 static bool hypercall_cflush = false; 43 44 /* buffers in highmem or foreign pages cannot cross page boundaries */ dma_cache_maint(struct device * dev,dma_addr_t handle,size_t size,u32 op)45 static void dma_cache_maint(struct device *dev, dma_addr_t handle, 46 size_t size, u32 op) 47 { 48 struct gnttab_cache_flush cflush; 49 50 cflush.offset = xen_offset_in_page(handle); 51 cflush.op = op; 52 handle &= XEN_PAGE_MASK; 53 54 do { 55 cflush.a.dev_bus_addr = dma_to_phys(dev, handle); 56 57 if (size + cflush.offset > XEN_PAGE_SIZE) 58 cflush.length = XEN_PAGE_SIZE - cflush.offset; 59 else 60 cflush.length = size; 61 62 HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); 63 64 cflush.offset = 0; 65 handle += cflush.length; 66 size -= cflush.length; 67 } while (size); 68 } 69 70 /* 71 * Dom0 is mapped 1:1, and while the Linux page can span across multiple Xen 72 * pages, it is not possible for it to contain a mix of local and foreign Xen 73 * pages. Calling pfn_valid on a foreign mfn will always return false, so if 74 * pfn_valid returns true the pages is local and we can use the native 75 * dma-direct functions, otherwise we call the Xen specific version. 76 */ xen_dma_sync_for_cpu(struct device * dev,dma_addr_t handle,size_t size,enum dma_data_direction dir)77 void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, 78 size_t size, enum dma_data_direction dir) 79 { 80 if (dir != DMA_TO_DEVICE) 81 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); 82 } 83 xen_dma_sync_for_device(struct device * dev,dma_addr_t handle,size_t size,enum dma_data_direction dir)84 void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, 85 size_t size, enum dma_data_direction dir) 86 { 87 if (dir == DMA_FROM_DEVICE) 88 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); 89 else 90 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_CLEAN); 91 } 92 xen_arch_need_swiotlb(struct device * dev,phys_addr_t phys,dma_addr_t dev_addr)93 bool xen_arch_need_swiotlb(struct device *dev, 94 phys_addr_t phys, 95 dma_addr_t dev_addr) 96 { 97 unsigned int xen_pfn = XEN_PFN_DOWN(phys); 98 unsigned int bfn = XEN_PFN_DOWN(dma_to_phys(dev, dev_addr)); 99 100 /* 101 * The swiotlb buffer should be used if 102 * - Xen doesn't have the cache flush hypercall 103 * - The Linux page refers to foreign memory 104 * - The device doesn't support coherent DMA request 105 * 106 * The Linux page may be spanned acrros multiple Xen page, although 107 * it's not possible to have a mix of local and foreign Xen page. 108 * Furthermore, range_straddles_page_boundary is already checking 109 * if buffer is physically contiguous in the host RAM. 110 * 111 * Therefore we only need to check the first Xen page to know if we 112 * require a bounce buffer because the device doesn't support coherent 113 * memory and we are not able to flush the cache. 114 */ 115 return (!hypercall_cflush && (xen_pfn != bfn) && 116 !dev_is_dma_coherent(dev)); 117 } 118 xen_mm_init(void)119 static int __init xen_mm_init(void) 120 { 121 struct gnttab_cache_flush cflush; 122 int rc; 123 124 if (!xen_swiotlb_detect()) 125 return 0; 126 127 /* we can work with the default swiotlb */ 128 rc = swiotlb_init_late(swiotlb_size_or_default(), 129 xen_swiotlb_gfp(), NULL); 130 if (rc < 0) 131 return rc; 132 133 cflush.op = 0; 134 cflush.a.dev_bus_addr = 0; 135 cflush.offset = 0; 136 cflush.length = 0; 137 if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1) != -ENOSYS) 138 hypercall_cflush = true; 139 return 0; 140 } 141 arch_initcall(xen_mm_init); 142