1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright (c) 2020 Intel Corporation. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/module.h> 10 11 #include "uverbs.h" 12 13 MODULE_IMPORT_NS(DMA_BUF); 14 15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 16 { 17 struct sg_table *sgt; 18 struct scatterlist *sg; 19 unsigned long start, end, cur = 0; 20 unsigned int nmap = 0; 21 long ret; 22 int i; 23 24 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 25 26 if (umem_dmabuf->sgt) 27 goto wait_fence; 28 29 sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL); 30 if (IS_ERR(sgt)) 31 return PTR_ERR(sgt); 32 33 /* modify the sg list in-place to match umem address and length */ 34 35 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); 36 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, 37 PAGE_SIZE); 38 for_each_sgtable_dma_sg(sgt, sg, i) { 39 if (start < cur + sg_dma_len(sg) && cur < end) 40 nmap++; 41 if (cur <= start && start < cur + sg_dma_len(sg)) { 42 unsigned long offset = start - cur; 43 44 umem_dmabuf->first_sg = sg; 45 umem_dmabuf->first_sg_offset = offset; 46 sg_dma_address(sg) += offset; 47 sg_dma_len(sg) -= offset; 48 cur += offset; 49 } 50 if (cur < end && end <= cur + sg_dma_len(sg)) { 51 unsigned long trim = cur + sg_dma_len(sg) - end; 52 53 umem_dmabuf->last_sg = sg; 54 umem_dmabuf->last_sg_trim = trim; 55 sg_dma_len(sg) -= trim; 56 break; 57 } 58 cur += sg_dma_len(sg); 59 } 60 61 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; 62 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; 63 umem_dmabuf->sgt = sgt; 64 65 wait_fence: 66 /* 67 * Although the sg list is valid now, the content of the pages 68 * may be not up-to-date. Wait for the exporter to finish 69 * the migration. 70 */ 71 ret = dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv, 72 DMA_RESV_USAGE_KERNEL, 73 false, MAX_SCHEDULE_TIMEOUT); 74 if (ret < 0) 75 return ret; 76 if (ret == 0) 77 return -ETIMEDOUT; 78 return 0; 79 } 80 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages); 81 82 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) 83 { 84 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 85 86 if (!umem_dmabuf->sgt) 87 return; 88 89 /* retore the original sg list */ 90 if (umem_dmabuf->first_sg) { 91 sg_dma_address(umem_dmabuf->first_sg) -= 92 umem_dmabuf->first_sg_offset; 93 sg_dma_len(umem_dmabuf->first_sg) += 94 umem_dmabuf->first_sg_offset; 95 umem_dmabuf->first_sg = NULL; 96 umem_dmabuf->first_sg_offset = 0; 97 } 98 if (umem_dmabuf->last_sg) { 99 sg_dma_len(umem_dmabuf->last_sg) += 100 umem_dmabuf->last_sg_trim; 101 umem_dmabuf->last_sg = NULL; 102 umem_dmabuf->last_sg_trim = 0; 103 } 104 105 dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt, 106 DMA_BIDIRECTIONAL); 107 108 umem_dmabuf->sgt = NULL; 109 } 110 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages); 111 112 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 113 unsigned long offset, size_t size, 114 int fd, int access, 115 const struct dma_buf_attach_ops *ops) 116 { 117 struct dma_buf *dmabuf; 118 struct ib_umem_dmabuf *umem_dmabuf; 119 struct ib_umem *umem; 120 unsigned long end; 121 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL); 122 123 if (check_add_overflow(offset, (unsigned long)size, &end)) 124 return ret; 125 126 if (unlikely(!ops || !ops->move_notify)) 127 return ret; 128 129 dmabuf = dma_buf_get(fd); 130 if (IS_ERR(dmabuf)) 131 return ERR_CAST(dmabuf); 132 133 if (dmabuf->size < end) 134 goto out_release_dmabuf; 135 136 umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL); 137 if (!umem_dmabuf) { 138 ret = ERR_PTR(-ENOMEM); 139 goto out_release_dmabuf; 140 } 141 142 umem = &umem_dmabuf->umem; 143 umem->ibdev = device; 144 umem->length = size; 145 umem->address = offset; 146 umem->writable = ib_access_writable(access); 147 umem->is_dmabuf = 1; 148 149 if (!ib_umem_num_pages(umem)) 150 goto out_free_umem; 151 152 umem_dmabuf->attach = dma_buf_dynamic_attach( 153 dmabuf, 154 device->dma_device, 155 ops, 156 umem_dmabuf); 157 if (IS_ERR(umem_dmabuf->attach)) { 158 ret = ERR_CAST(umem_dmabuf->attach); 159 goto out_free_umem; 160 } 161 return umem_dmabuf; 162 163 out_free_umem: 164 kfree(umem_dmabuf); 165 166 out_release_dmabuf: 167 dma_buf_put(dmabuf); 168 return ret; 169 } 170 EXPORT_SYMBOL(ib_umem_dmabuf_get); 171 172 static void 173 ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach) 174 { 175 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 176 177 ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev, 178 "Invalidate callback should not be called when memory is pinned\n"); 179 } 180 181 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { 182 .allow_peer2peer = true, 183 .move_notify = ib_umem_dmabuf_unsupported_move_notify, 184 }; 185 186 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 187 unsigned long offset, 188 size_t size, int fd, 189 int access) 190 { 191 struct ib_umem_dmabuf *umem_dmabuf; 192 int err; 193 194 umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access, 195 &ib_umem_dmabuf_attach_pinned_ops); 196 if (IS_ERR(umem_dmabuf)) 197 return umem_dmabuf; 198 199 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 200 err = dma_buf_pin(umem_dmabuf->attach); 201 if (err) 202 goto err_release; 203 umem_dmabuf->pinned = 1; 204 205 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 206 if (err) 207 goto err_unpin; 208 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 209 210 return umem_dmabuf; 211 212 err_unpin: 213 dma_buf_unpin(umem_dmabuf->attach); 214 err_release: 215 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 216 ib_umem_release(&umem_dmabuf->umem); 217 return ERR_PTR(err); 218 } 219 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned); 220 221 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) 222 { 223 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 224 225 dma_resv_lock(dmabuf->resv, NULL); 226 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 227 if (umem_dmabuf->pinned) 228 dma_buf_unpin(umem_dmabuf->attach); 229 dma_resv_unlock(dmabuf->resv); 230 231 dma_buf_detach(dmabuf, umem_dmabuf->attach); 232 dma_buf_put(dmabuf); 233 kfree(umem_dmabuf); 234 } 235