1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/cpu.h> 3 #include <linux/dma-direct.h> 4 #include <linux/dma-noncoherent.h> 5 #include <linux/gfp.h> 6 #include <linux/highmem.h> 7 #include <linux/export.h> 8 #include <linux/memblock.h> 9 #include <linux/of_address.h> 10 #include <linux/slab.h> 11 #include <linux/types.h> 12 #include <linux/vmalloc.h> 13 #include <linux/swiotlb.h> 14 15 #include <xen/xen.h> 16 #include <xen/interface/grant_table.h> 17 #include <xen/interface/memory.h> 18 #include <xen/page.h> 19 #include <xen/xen-ops.h> 20 #include <xen/swiotlb-xen.h> 21 22 #include <asm/cacheflush.h> 23 #include <asm/xen/hypercall.h> 24 #include <asm/xen/interface.h> 25 26 unsigned long xen_get_swiotlb_free_pages(unsigned int order) 27 { 28 struct memblock_region *reg; 29 gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM; 30 31 for_each_memblock(memory, reg) { 32 if (reg->base < (phys_addr_t)0xffffffff) { 33 if (IS_ENABLED(CONFIG_ZONE_DMA32)) 34 flags |= __GFP_DMA32; 35 else 36 flags |= __GFP_DMA; 37 break; 38 } 39 } 40 return __get_free_pages(flags, order); 41 } 42 43 static bool hypercall_cflush = false; 44 45 /* buffers in highmem or foreign pages cannot cross page boundaries */ 46 static void dma_cache_maint(struct device *dev, dma_addr_t handle, 47 size_t size, u32 op) 48 { 49 struct gnttab_cache_flush cflush; 50 51 cflush.offset = xen_offset_in_page(handle); 52 cflush.op = op; 53 handle &= XEN_PAGE_MASK; 54 55 do { 56 cflush.a.dev_bus_addr = dma_to_phys(dev, handle); 57 58 if (size + cflush.offset > XEN_PAGE_SIZE) 59 cflush.length = XEN_PAGE_SIZE - cflush.offset; 60 else 61 cflush.length = size; 62 63 HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); 64 65 cflush.offset = 0; 66 handle += cflush.length; 67 size -= cflush.length; 68 } while (size); 69 } 70 71 /* 72 * Dom0 is mapped 1:1, and while the Linux page can span across multiple Xen 73 * pages, it is not possible for it to contain a mix of local and foreign Xen 74 * pages. Calling pfn_valid on a foreign mfn will always return false, so if 75 * pfn_valid returns true the pages is local and we can use the native 76 * dma-direct functions, otherwise we call the Xen specific version. 77 */ 78 void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, 79 size_t size, enum dma_data_direction dir) 80 { 81 if (dir != DMA_TO_DEVICE) 82 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); 83 } 84 85 void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, 86 size_t size, enum dma_data_direction dir) 87 { 88 if (dir == DMA_FROM_DEVICE) 89 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); 90 else 91 dma_cache_maint(dev, handle, size, GNTTAB_CACHE_CLEAN); 92 } 93 94 bool xen_arch_need_swiotlb(struct device *dev, 95 phys_addr_t phys, 96 dma_addr_t dev_addr) 97 { 98 unsigned int xen_pfn = XEN_PFN_DOWN(phys); 99 unsigned int bfn = XEN_PFN_DOWN(dma_to_phys(dev, dev_addr)); 100 101 /* 102 * The swiotlb buffer should be used if 103 * - Xen doesn't have the cache flush hypercall 104 * - The Linux page refers to foreign memory 105 * - The device doesn't support coherent DMA request 106 * 107 * The Linux page may be spanned acrros multiple Xen page, although 108 * it's not possible to have a mix of local and foreign Xen page. 109 * Furthermore, range_straddles_page_boundary is already checking 110 * if buffer is physically contiguous in the host RAM. 111 * 112 * Therefore we only need to check the first Xen page to know if we 113 * require a bounce buffer because the device doesn't support coherent 114 * memory and we are not able to flush the cache. 115 */ 116 return (!hypercall_cflush && (xen_pfn != bfn) && 117 !dev_is_dma_coherent(dev)); 118 } 119 120 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, 121 unsigned int address_bits, 122 dma_addr_t *dma_handle) 123 { 124 if (!xen_initial_domain()) 125 return -EINVAL; 126 127 /* we assume that dom0 is mapped 1:1 for now */ 128 *dma_handle = pstart; 129 return 0; 130 } 131 132 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) 133 { 134 return; 135 } 136 137 static int __init xen_mm_init(void) 138 { 139 struct gnttab_cache_flush cflush; 140 if (!xen_initial_domain()) 141 return 0; 142 xen_swiotlb_init(1, false); 143 144 cflush.op = 0; 145 cflush.a.dev_bus_addr = 0; 146 cflush.offset = 0; 147 cflush.length = 0; 148 if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1) != -ENOSYS) 149 hypercall_cflush = true; 150 return 0; 151 } 152 arch_initcall(xen_mm_init); 153