1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/mm.h> 5 #include <linux/iosys-map.h> 6 #include <linux/sync_file.h> 7 #include <linux/pagemap.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/dma-mapping.h> 10 11 #include <drm/drm_file.h> 12 #include <drm/drm_syncobj.h> 13 #include <drm/drm_utils.h> 14 15 #include <drm/lima_drm.h> 16 17 #include "lima_drv.h" 18 #include "lima_gem.h" 19 #include "lima_vm.h" 20 21 int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) 22 { 23 struct page **pages; 24 struct address_space *mapping = bo->base.base.filp->f_mapping; 25 struct device *dev = bo->base.base.dev->dev; 26 size_t old_size = bo->heap_size; 27 size_t new_size = bo->heap_size ? bo->heap_size * 2 : 28 (lima_heap_init_nr_pages << PAGE_SHIFT); 29 struct sg_table sgt; 30 int i, ret; 31 32 if (bo->heap_size >= bo->base.base.size) 33 return -ENOSPC; 34 35 new_size = min(new_size, bo->base.base.size); 36 37 mutex_lock(&bo->base.pages_lock); 38 39 if (bo->base.pages) { 40 pages = bo->base.pages; 41 } else { 42 pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, 43 sizeof(*pages), GFP_KERNEL | __GFP_ZERO); 44 if (!pages) { 45 mutex_unlock(&bo->base.pages_lock); 46 return -ENOMEM; 47 } 48 49 bo->base.pages = pages; 50 bo->base.pages_use_count = 1; 51 52 mapping_set_unevictable(mapping); 53 } 54 55 for (i = old_size >> PAGE_SHIFT; i < new_size >> PAGE_SHIFT; i++) { 56 struct page *page = shmem_read_mapping_page(mapping, i); 57 58 if (IS_ERR(page)) { 59 mutex_unlock(&bo->base.pages_lock); 60 return PTR_ERR(page); 61 } 62 pages[i] = page; 63 } 64 65 mutex_unlock(&bo->base.pages_lock); 66 67 ret = sg_alloc_table_from_pages(&sgt, pages, i, 0, 68 new_size, GFP_KERNEL); 69 if (ret) 70 return ret; 71 72 if (bo->base.sgt) { 73 dma_unmap_sgtable(dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 74 sg_free_table(bo->base.sgt); 75 } else { 76 bo->base.sgt = kmalloc(sizeof(*bo->base.sgt), GFP_KERNEL); 77 if (!bo->base.sgt) { 78 sg_free_table(&sgt); 79 return -ENOMEM; 80 } 81 } 82 83 ret = dma_map_sgtable(dev, &sgt, DMA_BIDIRECTIONAL, 0); 84 if (ret) { 85 sg_free_table(&sgt); 86 kfree(bo->base.sgt); 87 bo->base.sgt = NULL; 88 return ret; 89 } 90 91 *bo->base.sgt = sgt; 92 93 if (vm) { 94 ret = lima_vm_map_bo(vm, bo, old_size >> PAGE_SHIFT); 95 if (ret) 96 return ret; 97 } 98 99 bo->heap_size = new_size; 100 return 0; 101 } 102 103 int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file, 104 u32 size, u32 flags, u32 *handle) 105 { 106 int err; 107 gfp_t mask; 108 struct drm_gem_shmem_object *shmem; 109 struct drm_gem_object *obj; 110 struct lima_bo *bo; 111 bool is_heap = flags & LIMA_BO_FLAG_HEAP; 112 113 shmem = drm_gem_shmem_create(dev, size); 114 if (IS_ERR(shmem)) 115 return PTR_ERR(shmem); 116 117 obj = &shmem->base; 118 119 /* Mali Utgard GPU can only support 32bit address space */ 120 mask = mapping_gfp_mask(obj->filp->f_mapping); 121 mask &= ~__GFP_HIGHMEM; 122 mask |= __GFP_DMA32; 123 mapping_set_gfp_mask(obj->filp->f_mapping, mask); 124 125 if (is_heap) { 126 bo = to_lima_bo(obj); 127 err = lima_heap_alloc(bo, NULL); 128 if (err) 129 goto out; 130 } else { 131 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(shmem); 132 133 if (IS_ERR(sgt)) { 134 err = PTR_ERR(sgt); 135 goto out; 136 } 137 } 138 139 err = drm_gem_handle_create(file, obj, handle); 140 141 out: 142 /* drop reference from allocate - handle holds it now */ 143 drm_gem_object_put(obj); 144 145 return err; 146 } 147 148 static void lima_gem_free_object(struct drm_gem_object *obj) 149 { 150 struct lima_bo *bo = to_lima_bo(obj); 151 152 if (!list_empty(&bo->va)) 153 dev_err(obj->dev->dev, "lima gem free bo still has va\n"); 154 155 drm_gem_shmem_free(&bo->base); 156 } 157 158 static int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file) 159 { 160 struct lima_bo *bo = to_lima_bo(obj); 161 struct lima_drm_priv *priv = to_lima_drm_priv(file); 162 struct lima_vm *vm = priv->vm; 163 164 return lima_vm_bo_add(vm, bo, true); 165 } 166 167 static void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file) 168 { 169 struct lima_bo *bo = to_lima_bo(obj); 170 struct lima_drm_priv *priv = to_lima_drm_priv(file); 171 struct lima_vm *vm = priv->vm; 172 173 lima_vm_bo_del(vm, bo); 174 } 175 176 static int lima_gem_pin(struct drm_gem_object *obj) 177 { 178 struct lima_bo *bo = to_lima_bo(obj); 179 180 if (bo->heap_size) 181 return -EINVAL; 182 183 return drm_gem_shmem_pin(&bo->base); 184 } 185 186 static int lima_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) 187 { 188 struct lima_bo *bo = to_lima_bo(obj); 189 190 if (bo->heap_size) 191 return -EINVAL; 192 193 return drm_gem_shmem_vmap(&bo->base, map); 194 } 195 196 static int lima_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 197 { 198 struct lima_bo *bo = to_lima_bo(obj); 199 200 if (bo->heap_size) 201 return -EINVAL; 202 203 return drm_gem_shmem_mmap(&bo->base, vma); 204 } 205 206 static const struct drm_gem_object_funcs lima_gem_funcs = { 207 .free = lima_gem_free_object, 208 .open = lima_gem_object_open, 209 .close = lima_gem_object_close, 210 .print_info = drm_gem_shmem_object_print_info, 211 .pin = lima_gem_pin, 212 .unpin = drm_gem_shmem_object_unpin, 213 .get_sg_table = drm_gem_shmem_object_get_sg_table, 214 .vmap = lima_gem_vmap, 215 .vunmap = drm_gem_shmem_object_vunmap, 216 .mmap = lima_gem_mmap, 217 .vm_ops = &drm_gem_shmem_vm_ops, 218 }; 219 220 struct drm_gem_object *lima_gem_create_object(struct drm_device *dev, size_t size) 221 { 222 struct lima_bo *bo; 223 224 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 225 if (!bo) 226 return ERR_PTR(-ENOMEM); 227 228 mutex_init(&bo->lock); 229 INIT_LIST_HEAD(&bo->va); 230 bo->base.map_wc = true; 231 bo->base.base.funcs = &lima_gem_funcs; 232 233 return &bo->base.base; 234 } 235 236 int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset) 237 { 238 struct drm_gem_object *obj; 239 struct lima_bo *bo; 240 struct lima_drm_priv *priv = to_lima_drm_priv(file); 241 struct lima_vm *vm = priv->vm; 242 243 obj = drm_gem_object_lookup(file, handle); 244 if (!obj) 245 return -ENOENT; 246 247 bo = to_lima_bo(obj); 248 249 *va = lima_vm_get_va(vm, bo); 250 251 *offset = drm_vma_node_offset_addr(&obj->vma_node); 252 253 drm_gem_object_put(obj); 254 return 0; 255 } 256 257 static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo, 258 bool write, bool explicit) 259 { 260 int err = 0; 261 262 if (!write) { 263 err = dma_resv_reserve_shared(lima_bo_resv(bo), 1); 264 if (err) 265 return err; 266 } 267 268 /* explicit sync use user passed dep fence */ 269 if (explicit) 270 return 0; 271 272 return drm_sched_job_add_implicit_dependencies(&task->base, 273 &bo->base.base, 274 write); 275 } 276 277 static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit) 278 { 279 int i, err; 280 281 for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) { 282 struct dma_fence *fence = NULL; 283 284 if (!submit->in_sync[i]) 285 continue; 286 287 err = drm_syncobj_find_fence(file, submit->in_sync[i], 288 0, 0, &fence); 289 if (err) 290 return err; 291 292 err = drm_sched_job_add_dependency(&submit->task->base, fence); 293 if (err) { 294 dma_fence_put(fence); 295 return err; 296 } 297 } 298 299 return 0; 300 } 301 302 int lima_gem_submit(struct drm_file *file, struct lima_submit *submit) 303 { 304 int i, err = 0; 305 struct ww_acquire_ctx ctx; 306 struct lima_drm_priv *priv = to_lima_drm_priv(file); 307 struct lima_vm *vm = priv->vm; 308 struct drm_syncobj *out_sync = NULL; 309 struct dma_fence *fence; 310 struct lima_bo **bos = submit->lbos; 311 312 if (submit->out_sync) { 313 out_sync = drm_syncobj_find(file, submit->out_sync); 314 if (!out_sync) 315 return -ENOENT; 316 } 317 318 for (i = 0; i < submit->nr_bos; i++) { 319 struct drm_gem_object *obj; 320 struct lima_bo *bo; 321 322 obj = drm_gem_object_lookup(file, submit->bos[i].handle); 323 if (!obj) { 324 err = -ENOENT; 325 goto err_out0; 326 } 327 328 bo = to_lima_bo(obj); 329 330 /* increase refcnt of gpu va map to prevent unmapped when executing, 331 * will be decreased when task done 332 */ 333 err = lima_vm_bo_add(vm, bo, false); 334 if (err) { 335 drm_gem_object_put(obj); 336 goto err_out0; 337 } 338 339 bos[i] = bo; 340 } 341 342 err = drm_gem_lock_reservations((struct drm_gem_object **)bos, 343 submit->nr_bos, &ctx); 344 if (err) 345 goto err_out0; 346 347 err = lima_sched_task_init( 348 submit->task, submit->ctx->context + submit->pipe, 349 bos, submit->nr_bos, vm); 350 if (err) 351 goto err_out1; 352 353 err = lima_gem_add_deps(file, submit); 354 if (err) 355 goto err_out2; 356 357 for (i = 0; i < submit->nr_bos; i++) { 358 err = lima_gem_sync_bo( 359 submit->task, bos[i], 360 submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE, 361 submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE); 362 if (err) 363 goto err_out2; 364 } 365 366 fence = lima_sched_context_queue_task(submit->task); 367 368 for (i = 0; i < submit->nr_bos; i++) { 369 if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE) 370 dma_resv_add_excl_fence(lima_bo_resv(bos[i]), fence); 371 else 372 dma_resv_add_shared_fence(lima_bo_resv(bos[i]), fence); 373 } 374 375 drm_gem_unlock_reservations((struct drm_gem_object **)bos, 376 submit->nr_bos, &ctx); 377 378 for (i = 0; i < submit->nr_bos; i++) 379 drm_gem_object_put(&bos[i]->base.base); 380 381 if (out_sync) { 382 drm_syncobj_replace_fence(out_sync, fence); 383 drm_syncobj_put(out_sync); 384 } 385 386 dma_fence_put(fence); 387 388 return 0; 389 390 err_out2: 391 lima_sched_task_fini(submit->task); 392 err_out1: 393 drm_gem_unlock_reservations((struct drm_gem_object **)bos, 394 submit->nr_bos, &ctx); 395 err_out0: 396 for (i = 0; i < submit->nr_bos; i++) { 397 if (!bos[i]) 398 break; 399 lima_vm_bo_del(vm, bos[i]); 400 drm_gem_object_put(&bos[i]->base.base); 401 } 402 if (out_sync) 403 drm_syncobj_put(out_sync); 404 return err; 405 } 406 407 int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns) 408 { 409 bool write = op & LIMA_GEM_WAIT_WRITE; 410 long ret, timeout; 411 412 if (!op) 413 return 0; 414 415 timeout = drm_timeout_abs_to_jiffies(timeout_ns); 416 417 ret = drm_gem_dma_resv_wait(file, handle, write, timeout); 418 if (ret == -ETIME) 419 ret = timeout ? -ETIMEDOUT : -EBUSY; 420 421 return ret; 422 } 423