1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright (c) 2020 Intel Corporation. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/module.h> 10 11 #include "uverbs.h" 12 13 MODULE_IMPORT_NS(DMA_BUF); 14 15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 16 { 17 struct sg_table *sgt; 18 struct scatterlist *sg; 19 unsigned long start, end, cur = 0; 20 unsigned int nmap = 0; 21 int i; 22 23 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 24 25 if (umem_dmabuf->sgt) 26 goto wait_fence; 27 28 sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL); 29 if (IS_ERR(sgt)) 30 return PTR_ERR(sgt); 31 32 /* modify the sg list in-place to match umem address and length */ 33 34 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); 35 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, 36 PAGE_SIZE); 37 for_each_sgtable_dma_sg(sgt, sg, i) { 38 if (start < cur + sg_dma_len(sg) && cur < end) 39 nmap++; 40 if (cur <= start && start < cur + sg_dma_len(sg)) { 41 unsigned long offset = start - cur; 42 43 umem_dmabuf->first_sg = sg; 44 umem_dmabuf->first_sg_offset = offset; 45 sg_dma_address(sg) += offset; 46 sg_dma_len(sg) -= offset; 47 cur += offset; 48 } 49 if (cur < end && end <= cur + sg_dma_len(sg)) { 50 unsigned long trim = cur + sg_dma_len(sg) - end; 51 52 umem_dmabuf->last_sg = sg; 53 umem_dmabuf->last_sg_trim = trim; 54 sg_dma_len(sg) -= trim; 55 break; 56 } 57 cur += sg_dma_len(sg); 58 } 59 60 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; 61 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; 62 umem_dmabuf->sgt = sgt; 63 64 wait_fence: 65 /* 66 * Although the sg list is valid now, the content of the pages 67 * may be not up-to-date. Wait for the exporter to finish 68 * the migration. 69 */ 70 return dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv, false, 71 false, MAX_SCHEDULE_TIMEOUT); 72 } 73 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages); 74 75 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) 76 { 77 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 78 79 if (!umem_dmabuf->sgt) 80 return; 81 82 /* retore the original sg list */ 83 if (umem_dmabuf->first_sg) { 84 sg_dma_address(umem_dmabuf->first_sg) -= 85 umem_dmabuf->first_sg_offset; 86 sg_dma_len(umem_dmabuf->first_sg) += 87 umem_dmabuf->first_sg_offset; 88 umem_dmabuf->first_sg = NULL; 89 umem_dmabuf->first_sg_offset = 0; 90 } 91 if (umem_dmabuf->last_sg) { 92 sg_dma_len(umem_dmabuf->last_sg) += 93 umem_dmabuf->last_sg_trim; 94 umem_dmabuf->last_sg = NULL; 95 umem_dmabuf->last_sg_trim = 0; 96 } 97 98 dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt, 99 DMA_BIDIRECTIONAL); 100 101 umem_dmabuf->sgt = NULL; 102 } 103 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages); 104 105 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 106 unsigned long offset, size_t size, 107 int fd, int access, 108 const struct dma_buf_attach_ops *ops) 109 { 110 struct dma_buf *dmabuf; 111 struct ib_umem_dmabuf *umem_dmabuf; 112 struct ib_umem *umem; 113 unsigned long end; 114 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL); 115 116 if (check_add_overflow(offset, (unsigned long)size, &end)) 117 return ret; 118 119 if (unlikely(!ops || !ops->move_notify)) 120 return ret; 121 122 dmabuf = dma_buf_get(fd); 123 if (IS_ERR(dmabuf)) 124 return ERR_CAST(dmabuf); 125 126 if (dmabuf->size < end) 127 goto out_release_dmabuf; 128 129 umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL); 130 if (!umem_dmabuf) { 131 ret = ERR_PTR(-ENOMEM); 132 goto out_release_dmabuf; 133 } 134 135 umem = &umem_dmabuf->umem; 136 umem->ibdev = device; 137 umem->length = size; 138 umem->address = offset; 139 umem->writable = ib_access_writable(access); 140 umem->is_dmabuf = 1; 141 142 if (!ib_umem_num_pages(umem)) 143 goto out_free_umem; 144 145 umem_dmabuf->attach = dma_buf_dynamic_attach( 146 dmabuf, 147 device->dma_device, 148 ops, 149 umem_dmabuf); 150 if (IS_ERR(umem_dmabuf->attach)) { 151 ret = ERR_CAST(umem_dmabuf->attach); 152 goto out_free_umem; 153 } 154 return umem_dmabuf; 155 156 out_free_umem: 157 kfree(umem_dmabuf); 158 159 out_release_dmabuf: 160 dma_buf_put(dmabuf); 161 return ret; 162 } 163 EXPORT_SYMBOL(ib_umem_dmabuf_get); 164 165 static void 166 ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach) 167 { 168 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 169 170 ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev, 171 "Invalidate callback should not be called when memory is pinned\n"); 172 } 173 174 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { 175 .allow_peer2peer = true, 176 .move_notify = ib_umem_dmabuf_unsupported_move_notify, 177 }; 178 179 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 180 unsigned long offset, 181 size_t size, int fd, 182 int access) 183 { 184 struct ib_umem_dmabuf *umem_dmabuf; 185 int err; 186 187 umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access, 188 &ib_umem_dmabuf_attach_pinned_ops); 189 if (IS_ERR(umem_dmabuf)) 190 return umem_dmabuf; 191 192 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 193 err = dma_buf_pin(umem_dmabuf->attach); 194 if (err) 195 goto err_release; 196 umem_dmabuf->pinned = 1; 197 198 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 199 if (err) 200 goto err_unpin; 201 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 202 203 return umem_dmabuf; 204 205 err_unpin: 206 dma_buf_unpin(umem_dmabuf->attach); 207 err_release: 208 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 209 ib_umem_release(&umem_dmabuf->umem); 210 return ERR_PTR(err); 211 } 212 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned); 213 214 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) 215 { 216 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 217 218 dma_resv_lock(dmabuf->resv, NULL); 219 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 220 if (umem_dmabuf->pinned) 221 dma_buf_unpin(umem_dmabuf->attach); 222 dma_resv_unlock(dmabuf->resv); 223 224 dma_buf_detach(dmabuf, umem_dmabuf->attach); 225 dma_buf_put(dmabuf); 226 kfree(umem_dmabuf); 227 } 228