1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright 2012 Red Hat Inc 5 */ 6 7 #include <linux/dma-buf.h> 8 #include <linux/highmem.h> 9 #include <linux/dma-resv.h> 10 11 #include "i915_drv.h" 12 #include "i915_gem_object.h" 13 #include "i915_scatterlist.h" 14 15 I915_SELFTEST_DECLARE(static bool force_different_devices;) 16 17 static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf) 18 { 19 return to_intel_bo(buf->priv); 20 } 21 22 static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment, 23 enum dma_data_direction dir) 24 { 25 struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); 26 struct sg_table *st; 27 struct scatterlist *src, *dst; 28 int ret, i; 29 30 /* Copy sg so that we make an independent mapping */ 31 st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); 32 if (st == NULL) { 33 ret = -ENOMEM; 34 goto err; 35 } 36 37 ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); 38 if (ret) 39 goto err_free; 40 41 src = obj->mm.pages->sgl; 42 dst = st->sgl; 43 for (i = 0; i < obj->mm.pages->nents; i++) { 44 sg_set_page(dst, sg_page(src), src->length, 0); 45 dst = sg_next(dst); 46 src = sg_next(src); 47 } 48 49 ret = dma_map_sgtable(attachment->dev, st, dir, DMA_ATTR_SKIP_CPU_SYNC); 50 if (ret) 51 goto err_free_sg; 52 53 return st; 54 55 err_free_sg: 56 sg_free_table(st); 57 err_free: 58 kfree(st); 59 err: 60 return ERR_PTR(ret); 61 } 62 63 static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, 64 struct sg_table *sg, 65 enum dma_data_direction dir) 66 { 67 dma_unmap_sgtable(attachment->dev, sg, dir, DMA_ATTR_SKIP_CPU_SYNC); 68 sg_free_table(sg); 69 kfree(sg); 70 } 71 72 static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map) 73 { 74 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); 75 void *vaddr; 76 77 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 78 if (IS_ERR(vaddr)) 79 return PTR_ERR(vaddr); 80 81 dma_buf_map_set_vaddr(map, vaddr); 82 83 return 0; 84 } 85 86 static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, struct dma_buf_map *map) 87 { 88 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); 89 90 i915_gem_object_flush_map(obj); 91 i915_gem_object_unpin_map(obj); 92 } 93 94 static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) 95 { 96 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); 97 int ret; 98 99 if (obj->base.size < vma->vm_end - vma->vm_start) 100 return -EINVAL; 101 102 if (!obj->base.filp) 103 return -ENODEV; 104 105 ret = call_mmap(obj->base.filp, vma); 106 if (ret) 107 return ret; 108 109 vma_set_file(vma, obj->base.filp); 110 111 return 0; 112 } 113 114 static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) 115 { 116 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); 117 bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE); 118 struct i915_gem_ww_ctx ww; 119 int err; 120 121 i915_gem_ww_ctx_init(&ww, true); 122 retry: 123 err = i915_gem_object_lock(obj, &ww); 124 if (!err) 125 err = i915_gem_object_pin_pages(obj); 126 if (!err) { 127 err = i915_gem_object_set_to_cpu_domain(obj, write); 128 i915_gem_object_unpin_pages(obj); 129 } 130 if (err == -EDEADLK) { 131 err = i915_gem_ww_ctx_backoff(&ww); 132 if (!err) 133 goto retry; 134 } 135 i915_gem_ww_ctx_fini(&ww); 136 return err; 137 } 138 139 static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) 140 { 141 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); 142 struct i915_gem_ww_ctx ww; 143 int err; 144 145 i915_gem_ww_ctx_init(&ww, true); 146 retry: 147 err = i915_gem_object_lock(obj, &ww); 148 if (!err) 149 err = i915_gem_object_pin_pages(obj); 150 if (!err) { 151 err = i915_gem_object_set_to_gtt_domain(obj, false); 152 i915_gem_object_unpin_pages(obj); 153 } 154 if (err == -EDEADLK) { 155 err = i915_gem_ww_ctx_backoff(&ww); 156 if (!err) 157 goto retry; 158 } 159 i915_gem_ww_ctx_fini(&ww); 160 return err; 161 } 162 163 static int i915_gem_dmabuf_attach(struct dma_buf *dmabuf, 164 struct dma_buf_attachment *attach) 165 { 166 struct drm_i915_gem_object *obj = dma_buf_to_obj(dmabuf); 167 struct i915_gem_ww_ctx ww; 168 int err; 169 170 if (!i915_gem_object_can_migrate(obj, INTEL_REGION_SMEM)) 171 return -EOPNOTSUPP; 172 173 for_i915_gem_ww(&ww, err, true) { 174 err = i915_gem_object_lock(obj, &ww); 175 if (err) 176 continue; 177 178 err = i915_gem_object_migrate(obj, &ww, INTEL_REGION_SMEM); 179 if (err) 180 continue; 181 182 err = i915_gem_object_wait_migration(obj, 0); 183 if (err) 184 continue; 185 186 err = i915_gem_object_pin_pages(obj); 187 } 188 189 return err; 190 } 191 192 static void i915_gem_dmabuf_detach(struct dma_buf *dmabuf, 193 struct dma_buf_attachment *attach) 194 { 195 struct drm_i915_gem_object *obj = dma_buf_to_obj(dmabuf); 196 197 i915_gem_object_unpin_pages(obj); 198 } 199 200 static const struct dma_buf_ops i915_dmabuf_ops = { 201 .attach = i915_gem_dmabuf_attach, 202 .detach = i915_gem_dmabuf_detach, 203 .map_dma_buf = i915_gem_map_dma_buf, 204 .unmap_dma_buf = i915_gem_unmap_dma_buf, 205 .release = drm_gem_dmabuf_release, 206 .mmap = i915_gem_dmabuf_mmap, 207 .vmap = i915_gem_dmabuf_vmap, 208 .vunmap = i915_gem_dmabuf_vunmap, 209 .begin_cpu_access = i915_gem_begin_cpu_access, 210 .end_cpu_access = i915_gem_end_cpu_access, 211 }; 212 213 struct dma_buf *i915_gem_prime_export(struct drm_gem_object *gem_obj, int flags) 214 { 215 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 216 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 217 218 exp_info.ops = &i915_dmabuf_ops; 219 exp_info.size = gem_obj->size; 220 exp_info.flags = flags; 221 exp_info.priv = gem_obj; 222 exp_info.resv = obj->base.resv; 223 224 if (obj->ops->dmabuf_export) { 225 int ret = obj->ops->dmabuf_export(obj); 226 if (ret) 227 return ERR_PTR(ret); 228 } 229 230 return drm_gem_dmabuf_export(gem_obj->dev, &exp_info); 231 } 232 233 static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) 234 { 235 struct sg_table *pages; 236 unsigned int sg_page_sizes; 237 238 assert_object_held(obj); 239 240 pages = dma_buf_map_attachment(obj->base.import_attach, 241 DMA_BIDIRECTIONAL); 242 if (IS_ERR(pages)) 243 return PTR_ERR(pages); 244 245 sg_page_sizes = i915_sg_dma_sizes(pages->sgl); 246 247 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 248 249 return 0; 250 } 251 252 static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj, 253 struct sg_table *pages) 254 { 255 dma_buf_unmap_attachment(obj->base.import_attach, pages, 256 DMA_BIDIRECTIONAL); 257 } 258 259 static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { 260 .name = "i915_gem_object_dmabuf", 261 .get_pages = i915_gem_object_get_pages_dmabuf, 262 .put_pages = i915_gem_object_put_pages_dmabuf, 263 }; 264 265 struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, 266 struct dma_buf *dma_buf) 267 { 268 static struct lock_class_key lock_class; 269 struct dma_buf_attachment *attach; 270 struct drm_i915_gem_object *obj; 271 int ret; 272 273 /* is this one of own objects? */ 274 if (dma_buf->ops == &i915_dmabuf_ops) { 275 obj = dma_buf_to_obj(dma_buf); 276 /* is it from our device? */ 277 if (obj->base.dev == dev && 278 !I915_SELFTEST_ONLY(force_different_devices)) { 279 /* 280 * Importing dmabuf exported from out own gem increases 281 * refcount on gem itself instead of f_count of dmabuf. 282 */ 283 return &i915_gem_object_get(obj)->base; 284 } 285 } 286 287 if (i915_gem_object_size_2big(dma_buf->size)) 288 return ERR_PTR(-E2BIG); 289 290 /* need to attach */ 291 attach = dma_buf_attach(dma_buf, dev->dev); 292 if (IS_ERR(attach)) 293 return ERR_CAST(attach); 294 295 get_dma_buf(dma_buf); 296 297 obj = i915_gem_object_alloc(); 298 if (obj == NULL) { 299 ret = -ENOMEM; 300 goto fail_detach; 301 } 302 303 drm_gem_private_object_init(dev, &obj->base, dma_buf->size); 304 i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class, 0); 305 obj->base.import_attach = attach; 306 obj->base.resv = dma_buf->resv; 307 308 /* We use GTT as shorthand for a coherent domain, one that is 309 * neither in the GPU cache nor in the CPU cache, where all 310 * writes are immediately visible in memory. (That's not strictly 311 * true, but it's close! There are internal buffers such as the 312 * write-combined buffer or a delay through the chipset for GTT 313 * writes that do require us to treat GTT as a separate cache domain.) 314 */ 315 obj->read_domains = I915_GEM_DOMAIN_GTT; 316 obj->write_domain = 0; 317 318 return &obj->base; 319 320 fail_detach: 321 dma_buf_detach(dma_buf, attach); 322 dma_buf_put(dma_buf); 323 324 return ERR_PTR(ret); 325 } 326 327 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 328 #include "selftests/mock_dmabuf.c" 329 #include "selftests/i915_gem_dmabuf.c" 330 #endif 331