1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/cpu.h> 3 #include <linux/dma-noncoherent.h> 4 #include <linux/gfp.h> 5 #include <linux/highmem.h> 6 #include <linux/export.h> 7 #include <linux/memblock.h> 8 #include <linux/of_address.h> 9 #include <linux/slab.h> 10 #include <linux/types.h> 11 #include <linux/vmalloc.h> 12 #include <linux/swiotlb.h> 13 14 #include <xen/xen.h> 15 #include <xen/interface/grant_table.h> 16 #include <xen/interface/memory.h> 17 #include <xen/page.h> 18 #include <xen/swiotlb-xen.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/xen/hypercall.h> 22 #include <asm/xen/interface.h> 23 24 unsigned long xen_get_swiotlb_free_pages(unsigned int order) 25 { 26 struct memblock_region *reg; 27 gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM; 28 29 for_each_memblock(memory, reg) { 30 if (reg->base < (phys_addr_t)0xffffffff) { 31 flags |= __GFP_DMA; 32 break; 33 } 34 } 35 return __get_free_pages(flags, order); 36 } 37 38 static bool hypercall_cflush = false; 39 40 /* buffers in highmem or foreign pages cannot cross page boundaries */ 41 static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) 42 { 43 struct gnttab_cache_flush cflush; 44 45 cflush.a.dev_bus_addr = handle & XEN_PAGE_MASK; 46 cflush.offset = xen_offset_in_page(handle); 47 cflush.op = op; 48 49 do { 50 if (size + cflush.offset > XEN_PAGE_SIZE) 51 cflush.length = XEN_PAGE_SIZE - cflush.offset; 52 else 53 cflush.length = size; 54 55 HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); 56 57 cflush.offset = 0; 58 cflush.a.dev_bus_addr += cflush.length; 59 size -= cflush.length; 60 } while (size); 61 } 62 63 /* 64 * Dom0 is mapped 1:1, and while the Linux page can span across multiple Xen 65 * pages, it is not possible for it to contain a mix of local and foreign Xen 66 * pages. Calling pfn_valid on a foreign mfn will always return false, so if 67 * pfn_valid returns true the pages is local and we can use the native 68 * dma-direct functions, otherwise we call the Xen specific version. 69 */ 70 void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, 71 phys_addr_t paddr, size_t size, enum dma_data_direction dir) 72 { 73 if (pfn_valid(PFN_DOWN(handle))) 74 arch_sync_dma_for_cpu(dev, paddr, size, dir); 75 else if (dir != DMA_TO_DEVICE) 76 dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); 77 } 78 79 void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, 80 phys_addr_t paddr, size_t size, enum dma_data_direction dir) 81 { 82 if (pfn_valid(PFN_DOWN(handle))) 83 arch_sync_dma_for_device(dev, paddr, size, dir); 84 else if (dir == DMA_FROM_DEVICE) 85 dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); 86 else 87 dma_cache_maint(handle, size, GNTTAB_CACHE_CLEAN); 88 } 89 90 bool xen_arch_need_swiotlb(struct device *dev, 91 phys_addr_t phys, 92 dma_addr_t dev_addr) 93 { 94 unsigned int xen_pfn = XEN_PFN_DOWN(phys); 95 unsigned int bfn = XEN_PFN_DOWN(dev_addr); 96 97 /* 98 * The swiotlb buffer should be used if 99 * - Xen doesn't have the cache flush hypercall 100 * - The Linux page refers to foreign memory 101 * - The device doesn't support coherent DMA request 102 * 103 * The Linux page may be spanned acrros multiple Xen page, although 104 * it's not possible to have a mix of local and foreign Xen page. 105 * Furthermore, range_straddles_page_boundary is already checking 106 * if buffer is physically contiguous in the host RAM. 107 * 108 * Therefore we only need to check the first Xen page to know if we 109 * require a bounce buffer because the device doesn't support coherent 110 * memory and we are not able to flush the cache. 111 */ 112 return (!hypercall_cflush && (xen_pfn != bfn) && 113 !dev_is_dma_coherent(dev)); 114 } 115 116 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, 117 unsigned int address_bits, 118 dma_addr_t *dma_handle) 119 { 120 if (!xen_initial_domain()) 121 return -EINVAL; 122 123 /* we assume that dom0 is mapped 1:1 for now */ 124 *dma_handle = pstart; 125 return 0; 126 } 127 128 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) 129 { 130 return; 131 } 132 133 int __init xen_mm_init(void) 134 { 135 struct gnttab_cache_flush cflush; 136 if (!xen_initial_domain()) 137 return 0; 138 xen_swiotlb_init(1, false); 139 140 cflush.op = 0; 141 cflush.a.dev_bus_addr = 0; 142 cflush.offset = 0; 143 cflush.length = 0; 144 if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1) != -ENOSYS) 145 hypercall_cflush = true; 146 return 0; 147 } 148 arch_initcall(xen_mm_init); 149