1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/mm.h> 5 #include <linux/sync_file.h> 6 #include <linux/pfn_t.h> 7 8 #include <drm/drm_file.h> 9 #include <drm/drm_syncobj.h> 10 #include <drm/drm_utils.h> 11 12 #include <drm/lima_drm.h> 13 14 #include "lima_drv.h" 15 #include "lima_gem.h" 16 #include "lima_gem_prime.h" 17 #include "lima_vm.h" 18 #include "lima_object.h" 19 20 int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file, 21 u32 size, u32 flags, u32 *handle) 22 { 23 int err; 24 struct lima_bo *bo; 25 struct lima_device *ldev = to_lima_dev(dev); 26 27 bo = lima_bo_create(ldev, size, flags, NULL); 28 if (IS_ERR(bo)) 29 return PTR_ERR(bo); 30 31 err = drm_gem_handle_create(file, &bo->gem, handle); 32 33 /* drop reference from allocate - handle holds it now */ 34 drm_gem_object_put_unlocked(&bo->gem); 35 36 return err; 37 } 38 39 void lima_gem_free_object(struct drm_gem_object *obj) 40 { 41 struct lima_bo *bo = to_lima_bo(obj); 42 43 if (!list_empty(&bo->va)) 44 dev_err(obj->dev->dev, "lima gem free bo still has va\n"); 45 46 lima_bo_destroy(bo); 47 } 48 49 int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file) 50 { 51 struct lima_bo *bo = to_lima_bo(obj); 52 struct lima_drm_priv *priv = to_lima_drm_priv(file); 53 struct lima_vm *vm = priv->vm; 54 55 return lima_vm_bo_add(vm, bo, true); 56 } 57 58 void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file) 59 { 60 struct lima_bo *bo = to_lima_bo(obj); 61 struct lima_drm_priv *priv = to_lima_drm_priv(file); 62 struct lima_vm *vm = priv->vm; 63 64 lima_vm_bo_del(vm, bo); 65 } 66 67 int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset) 68 { 69 struct drm_gem_object *obj; 70 struct lima_bo *bo; 71 struct lima_drm_priv *priv = to_lima_drm_priv(file); 72 struct lima_vm *vm = priv->vm; 73 int err; 74 75 obj = drm_gem_object_lookup(file, handle); 76 if (!obj) 77 return -ENOENT; 78 79 bo = to_lima_bo(obj); 80 81 *va = lima_vm_get_va(vm, bo); 82 83 err = drm_gem_create_mmap_offset(obj); 84 if (!err) 85 *offset = drm_vma_node_offset_addr(&obj->vma_node); 86 87 drm_gem_object_put_unlocked(obj); 88 return err; 89 } 90 91 static vm_fault_t lima_gem_fault(struct vm_fault *vmf) 92 { 93 struct vm_area_struct *vma = vmf->vma; 94 struct drm_gem_object *obj = vma->vm_private_data; 95 struct lima_bo *bo = to_lima_bo(obj); 96 pfn_t pfn; 97 pgoff_t pgoff; 98 99 /* We don't use vmf->pgoff since that has the fake offset: */ 100 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 101 pfn = __pfn_to_pfn_t(page_to_pfn(bo->pages[pgoff]), PFN_DEV); 102 103 return vmf_insert_mixed(vma, vmf->address, pfn); 104 } 105 106 const struct vm_operations_struct lima_gem_vm_ops = { 107 .fault = lima_gem_fault, 108 .open = drm_gem_vm_open, 109 .close = drm_gem_vm_close, 110 }; 111 112 void lima_set_vma_flags(struct vm_area_struct *vma) 113 { 114 pgprot_t prot = vm_get_page_prot(vma->vm_flags); 115 116 vma->vm_flags |= VM_MIXEDMAP; 117 vma->vm_flags &= ~VM_PFNMAP; 118 vma->vm_page_prot = pgprot_writecombine(prot); 119 } 120 121 int lima_gem_mmap(struct file *filp, struct vm_area_struct *vma) 122 { 123 int ret; 124 125 ret = drm_gem_mmap(filp, vma); 126 if (ret) 127 return ret; 128 129 lima_set_vma_flags(vma); 130 return 0; 131 } 132 133 static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo, 134 bool write, bool explicit) 135 { 136 int err = 0; 137 138 if (!write) { 139 err = dma_resv_reserve_shared(bo->gem.resv, 1); 140 if (err) 141 return err; 142 } 143 144 /* explicit sync use user passed dep fence */ 145 if (explicit) 146 return 0; 147 148 return drm_gem_fence_array_add_implicit(&task->deps, &bo->gem, write); 149 } 150 151 static int lima_gem_lock_bos(struct lima_bo **bos, u32 nr_bos, 152 struct ww_acquire_ctx *ctx) 153 { 154 int i, ret = 0, contended, slow_locked = -1; 155 156 ww_acquire_init(ctx, &reservation_ww_class); 157 158 retry: 159 for (i = 0; i < nr_bos; i++) { 160 if (i == slow_locked) { 161 slow_locked = -1; 162 continue; 163 } 164 165 ret = ww_mutex_lock_interruptible(&bos[i]->gem.resv->lock, ctx); 166 if (ret < 0) { 167 contended = i; 168 goto err; 169 } 170 } 171 172 ww_acquire_done(ctx); 173 return 0; 174 175 err: 176 for (i--; i >= 0; i--) 177 ww_mutex_unlock(&bos[i]->gem.resv->lock); 178 179 if (slow_locked >= 0) 180 ww_mutex_unlock(&bos[slow_locked]->gem.resv->lock); 181 182 if (ret == -EDEADLK) { 183 /* we lost out in a seqno race, lock and retry.. */ 184 ret = ww_mutex_lock_slow_interruptible( 185 &bos[contended]->gem.resv->lock, ctx); 186 if (!ret) { 187 slow_locked = contended; 188 goto retry; 189 } 190 } 191 ww_acquire_fini(ctx); 192 193 return ret; 194 } 195 196 static void lima_gem_unlock_bos(struct lima_bo **bos, u32 nr_bos, 197 struct ww_acquire_ctx *ctx) 198 { 199 int i; 200 201 for (i = 0; i < nr_bos; i++) 202 ww_mutex_unlock(&bos[i]->gem.resv->lock); 203 ww_acquire_fini(ctx); 204 } 205 206 static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit) 207 { 208 int i, err; 209 210 for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) { 211 struct dma_fence *fence = NULL; 212 213 if (!submit->in_sync[i]) 214 continue; 215 216 err = drm_syncobj_find_fence(file, submit->in_sync[i], 217 0, 0, &fence); 218 if (err) 219 return err; 220 221 err = drm_gem_fence_array_add(&submit->task->deps, fence); 222 if (err) { 223 dma_fence_put(fence); 224 return err; 225 } 226 } 227 228 return 0; 229 } 230 231 int lima_gem_submit(struct drm_file *file, struct lima_submit *submit) 232 { 233 int i, err = 0; 234 struct ww_acquire_ctx ctx; 235 struct lima_drm_priv *priv = to_lima_drm_priv(file); 236 struct lima_vm *vm = priv->vm; 237 struct drm_syncobj *out_sync = NULL; 238 struct dma_fence *fence; 239 struct lima_bo **bos = submit->lbos; 240 241 if (submit->out_sync) { 242 out_sync = drm_syncobj_find(file, submit->out_sync); 243 if (!out_sync) 244 return -ENOENT; 245 } 246 247 for (i = 0; i < submit->nr_bos; i++) { 248 struct drm_gem_object *obj; 249 struct lima_bo *bo; 250 251 obj = drm_gem_object_lookup(file, submit->bos[i].handle); 252 if (!obj) { 253 err = -ENOENT; 254 goto err_out0; 255 } 256 257 bo = to_lima_bo(obj); 258 259 /* increase refcnt of gpu va map to prevent unmapped when executing, 260 * will be decreased when task done 261 */ 262 err = lima_vm_bo_add(vm, bo, false); 263 if (err) { 264 drm_gem_object_put_unlocked(obj); 265 goto err_out0; 266 } 267 268 bos[i] = bo; 269 } 270 271 err = lima_gem_lock_bos(bos, submit->nr_bos, &ctx); 272 if (err) 273 goto err_out0; 274 275 err = lima_sched_task_init( 276 submit->task, submit->ctx->context + submit->pipe, 277 bos, submit->nr_bos, vm); 278 if (err) 279 goto err_out1; 280 281 err = lima_gem_add_deps(file, submit); 282 if (err) 283 goto err_out2; 284 285 for (i = 0; i < submit->nr_bos; i++) { 286 err = lima_gem_sync_bo( 287 submit->task, bos[i], 288 submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE, 289 submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE); 290 if (err) 291 goto err_out2; 292 } 293 294 fence = lima_sched_context_queue_task( 295 submit->ctx->context + submit->pipe, submit->task); 296 297 for (i = 0; i < submit->nr_bos; i++) { 298 if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE) 299 dma_resv_add_excl_fence(bos[i]->gem.resv, fence); 300 else 301 dma_resv_add_shared_fence(bos[i]->gem.resv, fence); 302 } 303 304 lima_gem_unlock_bos(bos, submit->nr_bos, &ctx); 305 306 for (i = 0; i < submit->nr_bos; i++) 307 drm_gem_object_put_unlocked(&bos[i]->gem); 308 309 if (out_sync) { 310 drm_syncobj_replace_fence(out_sync, fence); 311 drm_syncobj_put(out_sync); 312 } 313 314 dma_fence_put(fence); 315 316 return 0; 317 318 err_out2: 319 lima_sched_task_fini(submit->task); 320 err_out1: 321 lima_gem_unlock_bos(bos, submit->nr_bos, &ctx); 322 err_out0: 323 for (i = 0; i < submit->nr_bos; i++) { 324 if (!bos[i]) 325 break; 326 lima_vm_bo_del(vm, bos[i]); 327 drm_gem_object_put_unlocked(&bos[i]->gem); 328 } 329 if (out_sync) 330 drm_syncobj_put(out_sync); 331 return err; 332 } 333 334 int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns) 335 { 336 bool write = op & LIMA_GEM_WAIT_WRITE; 337 long ret, timeout; 338 339 if (!op) 340 return 0; 341 342 timeout = drm_timeout_abs_to_jiffies(timeout_ns); 343 344 ret = drm_gem_dma_resv_wait(file, handle, write, timeout); 345 if (ret == 0) 346 ret = timeout ? -ETIMEDOUT : -EBUSY; 347 348 return ret; 349 } 350