1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/cpu.h> 3 #include <linux/dma-noncoherent.h> 4 #include <linux/gfp.h> 5 #include <linux/highmem.h> 6 #include <linux/export.h> 7 #include <linux/memblock.h> 8 #include <linux/of_address.h> 9 #include <linux/slab.h> 10 #include <linux/types.h> 11 #include <linux/vmalloc.h> 12 #include <linux/swiotlb.h> 13 14 #include <xen/xen.h> 15 #include <xen/interface/grant_table.h> 16 #include <xen/interface/memory.h> 17 #include <xen/page.h> 18 #include <xen/xen-ops.h> 19 #include <xen/swiotlb-xen.h> 20 21 #include <asm/cacheflush.h> 22 #include <asm/xen/hypercall.h> 23 #include <asm/xen/interface.h> 24 25 unsigned long xen_get_swiotlb_free_pages(unsigned int order) 26 { 27 struct memblock_region *reg; 28 gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM; 29 30 for_each_memblock(memory, reg) { 31 if (reg->base < (phys_addr_t)0xffffffff) { 32 if (IS_ENABLED(CONFIG_ZONE_DMA32)) 33 flags |= __GFP_DMA32; 34 else 35 flags |= __GFP_DMA; 36 break; 37 } 38 } 39 return __get_free_pages(flags, order); 40 } 41 42 static bool hypercall_cflush = false; 43 44 /* buffers in highmem or foreign pages cannot cross page boundaries */ 45 static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) 46 { 47 struct gnttab_cache_flush cflush; 48 49 cflush.a.dev_bus_addr = handle & XEN_PAGE_MASK; 50 cflush.offset = xen_offset_in_page(handle); 51 cflush.op = op; 52 53 do { 54 if (size + cflush.offset > XEN_PAGE_SIZE) 55 cflush.length = XEN_PAGE_SIZE - cflush.offset; 56 else 57 cflush.length = size; 58 59 HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); 60 61 cflush.offset = 0; 62 cflush.a.dev_bus_addr += cflush.length; 63 size -= cflush.length; 64 } while (size); 65 } 66 67 /* 68 * Dom0 is mapped 1:1, and while the Linux page can span across multiple Xen 69 * pages, it is not possible for it to contain a mix of local and foreign Xen 70 * pages. Calling pfn_valid on a foreign mfn will always return false, so if 71 * pfn_valid returns true the pages is local and we can use the native 72 * dma-direct functions, otherwise we call the Xen specific version. 73 */ 74 void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size, 75 enum dma_data_direction dir) 76 { 77 if (pfn_valid(PFN_DOWN(handle))) 78 arch_sync_dma_for_cpu(paddr, size, dir); 79 else if (dir != DMA_TO_DEVICE) 80 dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); 81 } 82 83 void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, 84 enum dma_data_direction dir) 85 { 86 if (pfn_valid(PFN_DOWN(handle))) 87 arch_sync_dma_for_device(paddr, size, dir); 88 else if (dir == DMA_FROM_DEVICE) 89 dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); 90 else 91 dma_cache_maint(handle, size, GNTTAB_CACHE_CLEAN); 92 } 93 94 bool xen_arch_need_swiotlb(struct device *dev, 95 phys_addr_t phys, 96 dma_addr_t dev_addr) 97 { 98 unsigned int xen_pfn = XEN_PFN_DOWN(phys); 99 unsigned int bfn = XEN_PFN_DOWN(dev_addr); 100 101 /* 102 * The swiotlb buffer should be used if 103 * - Xen doesn't have the cache flush hypercall 104 * - The Linux page refers to foreign memory 105 * - The device doesn't support coherent DMA request 106 * 107 * The Linux page may be spanned acrros multiple Xen page, although 108 * it's not possible to have a mix of local and foreign Xen page. 109 * Furthermore, range_straddles_page_boundary is already checking 110 * if buffer is physically contiguous in the host RAM. 111 * 112 * Therefore we only need to check the first Xen page to know if we 113 * require a bounce buffer because the device doesn't support coherent 114 * memory and we are not able to flush the cache. 115 */ 116 return (!hypercall_cflush && (xen_pfn != bfn) && 117 !dev_is_dma_coherent(dev)); 118 } 119 120 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, 121 unsigned int address_bits, 122 dma_addr_t *dma_handle) 123 { 124 if (!xen_initial_domain()) 125 return -EINVAL; 126 127 /* we assume that dom0 is mapped 1:1 for now */ 128 *dma_handle = pstart; 129 return 0; 130 } 131 132 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) 133 { 134 return; 135 } 136 137 static int __init xen_mm_init(void) 138 { 139 struct gnttab_cache_flush cflush; 140 if (!xen_initial_domain()) 141 return 0; 142 xen_swiotlb_init(1, false); 143 144 cflush.op = 0; 145 cflush.a.dev_bus_addr = 0; 146 cflush.offset = 0; 147 cflush.length = 0; 148 if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1) != -ENOSYS) 149 hypercall_cflush = true; 150 return 0; 151 } 152 arch_initcall(xen_mm_init); 153