1d5b1a78aSEric Anholt /* 2d5b1a78aSEric Anholt * Copyright © 2014 Broadcom 3d5b1a78aSEric Anholt * 4d5b1a78aSEric Anholt * Permission is hereby granted, free of charge, to any person obtaining a 5d5b1a78aSEric Anholt * copy of this software and associated documentation files (the "Software"), 6d5b1a78aSEric Anholt * to deal in the Software without restriction, including without limitation 7d5b1a78aSEric Anholt * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8d5b1a78aSEric Anholt * and/or sell copies of the Software, and to permit persons to whom the 9d5b1a78aSEric Anholt * Software is furnished to do so, subject to the following conditions: 10d5b1a78aSEric Anholt * 11d5b1a78aSEric Anholt * The above copyright notice and this permission notice (including the next 12d5b1a78aSEric Anholt * paragraph) shall be included in all copies or substantial portions of the 13d5b1a78aSEric Anholt * Software. 14d5b1a78aSEric Anholt * 15d5b1a78aSEric Anholt * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16d5b1a78aSEric Anholt * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17d5b1a78aSEric Anholt * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18d5b1a78aSEric Anholt * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19d5b1a78aSEric Anholt * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20d5b1a78aSEric Anholt * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21d5b1a78aSEric Anholt * IN THE SOFTWARE. 22d5b1a78aSEric Anholt */ 23d5b1a78aSEric Anholt 24d5b1a78aSEric Anholt #include <linux/module.h> 25d5b1a78aSEric Anholt #include <linux/platform_device.h> 26001bdb55SEric Anholt #include <linux/pm_runtime.h> 27d5b1a78aSEric Anholt #include <linux/device.h> 28d5b1a78aSEric Anholt #include <linux/io.h> 29174cd4b1SIngo Molnar #include <linux/sched/signal.h> 30d5b1a78aSEric Anholt 31d5b1a78aSEric Anholt #include "uapi/drm/vc4_drm.h" 32d5b1a78aSEric Anholt #include "vc4_drv.h" 33d5b1a78aSEric Anholt #include "vc4_regs.h" 34d5b1a78aSEric Anholt #include "vc4_trace.h" 35d5b1a78aSEric Anholt 36d5b1a78aSEric Anholt static void 37d5b1a78aSEric Anholt vc4_queue_hangcheck(struct drm_device *dev) 38d5b1a78aSEric Anholt { 39d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 40d5b1a78aSEric Anholt 41d5b1a78aSEric Anholt mod_timer(&vc4->hangcheck.timer, 42d5b1a78aSEric Anholt round_jiffies_up(jiffies + msecs_to_jiffies(100))); 43d5b1a78aSEric Anholt } 44d5b1a78aSEric Anholt 4521461365SEric Anholt struct vc4_hang_state { 4621461365SEric Anholt struct drm_vc4_get_hang_state user_state; 4721461365SEric Anholt 4821461365SEric Anholt u32 bo_count; 4921461365SEric Anholt struct drm_gem_object **bo; 5021461365SEric Anholt }; 5121461365SEric Anholt 5221461365SEric Anholt static void 5321461365SEric Anholt vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state) 5421461365SEric Anholt { 5521461365SEric Anholt unsigned int i; 5621461365SEric Anholt 5721461365SEric Anholt for (i = 0; i < state->user_state.bo_count; i++) 581d5494e9SCihangir Akturk drm_gem_object_put_unlocked(state->bo[i]); 5921461365SEric Anholt 6021461365SEric Anholt kfree(state); 6121461365SEric Anholt } 6221461365SEric Anholt 6321461365SEric Anholt int 6421461365SEric Anholt vc4_get_hang_state_ioctl(struct drm_device *dev, void *data, 6521461365SEric Anholt struct drm_file *file_priv) 6621461365SEric Anholt { 6721461365SEric Anholt struct drm_vc4_get_hang_state *get_state = data; 6821461365SEric Anholt struct drm_vc4_get_hang_state_bo *bo_state; 6921461365SEric Anholt struct vc4_hang_state *kernel_state; 7021461365SEric Anholt struct drm_vc4_get_hang_state *state; 7121461365SEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 7221461365SEric Anholt unsigned long irqflags; 7321461365SEric Anholt u32 i; 7465c4777dSDan Carpenter int ret = 0; 7521461365SEric Anholt 7621461365SEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 7721461365SEric Anholt kernel_state = vc4->hang_state; 7821461365SEric Anholt if (!kernel_state) { 7921461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 8021461365SEric Anholt return -ENOENT; 8121461365SEric Anholt } 8221461365SEric Anholt state = &kernel_state->user_state; 8321461365SEric Anholt 8421461365SEric Anholt /* If the user's array isn't big enough, just return the 8521461365SEric Anholt * required array size. 8621461365SEric Anholt */ 8721461365SEric Anholt if (get_state->bo_count < state->bo_count) { 8821461365SEric Anholt get_state->bo_count = state->bo_count; 8921461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 9021461365SEric Anholt return 0; 9121461365SEric Anholt } 9221461365SEric Anholt 9321461365SEric Anholt vc4->hang_state = NULL; 9421461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 9521461365SEric Anholt 9621461365SEric Anholt /* Save the user's BO pointer, so we don't stomp it with the memcpy. */ 9721461365SEric Anholt state->bo = get_state->bo; 9821461365SEric Anholt memcpy(get_state, state, sizeof(*state)); 9921461365SEric Anholt 10021461365SEric Anholt bo_state = kcalloc(state->bo_count, sizeof(*bo_state), GFP_KERNEL); 10121461365SEric Anholt if (!bo_state) { 10221461365SEric Anholt ret = -ENOMEM; 10321461365SEric Anholt goto err_free; 10421461365SEric Anholt } 10521461365SEric Anholt 10621461365SEric Anholt for (i = 0; i < state->bo_count; i++) { 10721461365SEric Anholt struct vc4_bo *vc4_bo = to_vc4_bo(kernel_state->bo[i]); 10821461365SEric Anholt u32 handle; 10921461365SEric Anholt 11021461365SEric Anholt ret = drm_gem_handle_create(file_priv, kernel_state->bo[i], 11121461365SEric Anholt &handle); 11221461365SEric Anholt 11321461365SEric Anholt if (ret) { 114d0b1d259SChristophe JAILLET state->bo_count = i; 115d0b1d259SChristophe JAILLET goto err_delete_handle; 11621461365SEric Anholt } 11721461365SEric Anholt bo_state[i].handle = handle; 11821461365SEric Anholt bo_state[i].paddr = vc4_bo->base.paddr; 11921461365SEric Anholt bo_state[i].size = vc4_bo->base.base.size; 12021461365SEric Anholt } 12121461365SEric Anholt 12295d7cbcbSEric Anholt if (copy_to_user(u64_to_user_ptr(get_state->bo), 12321461365SEric Anholt bo_state, 12465c4777dSDan Carpenter state->bo_count * sizeof(*bo_state))) 12565c4777dSDan Carpenter ret = -EFAULT; 12665c4777dSDan Carpenter 127d0b1d259SChristophe JAILLET err_delete_handle: 128d0b1d259SChristophe JAILLET if (ret) { 129d0b1d259SChristophe JAILLET for (i = 0; i < state->bo_count; i++) 130d0b1d259SChristophe JAILLET drm_gem_handle_delete(file_priv, bo_state[i].handle); 131d0b1d259SChristophe JAILLET } 13221461365SEric Anholt 13321461365SEric Anholt err_free: 13421461365SEric Anholt vc4_free_hang_state(dev, kernel_state); 135d0b1d259SChristophe JAILLET kfree(bo_state); 13621461365SEric Anholt 13721461365SEric Anholt return ret; 13821461365SEric Anholt } 13921461365SEric Anholt 14021461365SEric Anholt static void 14121461365SEric Anholt vc4_save_hang_state(struct drm_device *dev) 14221461365SEric Anholt { 14321461365SEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 14421461365SEric Anholt struct drm_vc4_get_hang_state *state; 14521461365SEric Anholt struct vc4_hang_state *kernel_state; 146ca26d28bSVarad Gautam struct vc4_exec_info *exec[2]; 14721461365SEric Anholt struct vc4_bo *bo; 14821461365SEric Anholt unsigned long irqflags; 149ca26d28bSVarad Gautam unsigned int i, j, unref_list_count, prev_idx; 15021461365SEric Anholt 1517e5082fbSDan Carpenter kernel_state = kcalloc(1, sizeof(*kernel_state), GFP_KERNEL); 15221461365SEric Anholt if (!kernel_state) 15321461365SEric Anholt return; 15421461365SEric Anholt 15521461365SEric Anholt state = &kernel_state->user_state; 15621461365SEric Anholt 15721461365SEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 158ca26d28bSVarad Gautam exec[0] = vc4_first_bin_job(vc4); 159ca26d28bSVarad Gautam exec[1] = vc4_first_render_job(vc4); 160ca26d28bSVarad Gautam if (!exec[0] && !exec[1]) { 16121461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 16221461365SEric Anholt return; 16321461365SEric Anholt } 16421461365SEric Anholt 165ca26d28bSVarad Gautam /* Get the bos from both binner and renderer into hang state. */ 166ca26d28bSVarad Gautam state->bo_count = 0; 167ca26d28bSVarad Gautam for (i = 0; i < 2; i++) { 168ca26d28bSVarad Gautam if (!exec[i]) 169ca26d28bSVarad Gautam continue; 17021461365SEric Anholt 171ca26d28bSVarad Gautam unref_list_count = 0; 172ca26d28bSVarad Gautam list_for_each_entry(bo, &exec[i]->unref_list, unref_head) 173ca26d28bSVarad Gautam unref_list_count++; 174ca26d28bSVarad Gautam state->bo_count += exec[i]->bo_count + unref_list_count; 175ca26d28bSVarad Gautam } 176ca26d28bSVarad Gautam 177ca26d28bSVarad Gautam kernel_state->bo = kcalloc(state->bo_count, 178ca26d28bSVarad Gautam sizeof(*kernel_state->bo), GFP_ATOMIC); 179ca26d28bSVarad Gautam 18021461365SEric Anholt if (!kernel_state->bo) { 18121461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 18221461365SEric Anholt return; 18321461365SEric Anholt } 18421461365SEric Anholt 185ca26d28bSVarad Gautam prev_idx = 0; 186ca26d28bSVarad Gautam for (i = 0; i < 2; i++) { 187ca26d28bSVarad Gautam if (!exec[i]) 188ca26d28bSVarad Gautam continue; 189ca26d28bSVarad Gautam 190ca26d28bSVarad Gautam for (j = 0; j < exec[i]->bo_count; j++) { 191b9f19259SBoris Brezillon bo = to_vc4_bo(&exec[i]->bo[j]->base); 192b9f19259SBoris Brezillon 193b9f19259SBoris Brezillon /* Retain BOs just in case they were marked purgeable. 194b9f19259SBoris Brezillon * This prevents the BO from being purged before 195b9f19259SBoris Brezillon * someone had a chance to dump the hang state. 196b9f19259SBoris Brezillon */ 197b9f19259SBoris Brezillon WARN_ON(!refcount_read(&bo->usecnt)); 198b9f19259SBoris Brezillon refcount_inc(&bo->usecnt); 1991d5494e9SCihangir Akturk drm_gem_object_get(&exec[i]->bo[j]->base); 200ca26d28bSVarad Gautam kernel_state->bo[j + prev_idx] = &exec[i]->bo[j]->base; 20121461365SEric Anholt } 20221461365SEric Anholt 203ca26d28bSVarad Gautam list_for_each_entry(bo, &exec[i]->unref_list, unref_head) { 204b9f19259SBoris Brezillon /* No need to retain BOs coming from the ->unref_list 205b9f19259SBoris Brezillon * because they are naturally unpurgeable. 206b9f19259SBoris Brezillon */ 2071d5494e9SCihangir Akturk drm_gem_object_get(&bo->base.base); 208ca26d28bSVarad Gautam kernel_state->bo[j + prev_idx] = &bo->base.base; 209ca26d28bSVarad Gautam j++; 210ca26d28bSVarad Gautam } 211ca26d28bSVarad Gautam prev_idx = j + 1; 21221461365SEric Anholt } 21321461365SEric Anholt 214ca26d28bSVarad Gautam if (exec[0]) 215ca26d28bSVarad Gautam state->start_bin = exec[0]->ct0ca; 216ca26d28bSVarad Gautam if (exec[1]) 217ca26d28bSVarad Gautam state->start_render = exec[1]->ct1ca; 21821461365SEric Anholt 21921461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 22021461365SEric Anholt 22121461365SEric Anholt state->ct0ca = V3D_READ(V3D_CTNCA(0)); 22221461365SEric Anholt state->ct0ea = V3D_READ(V3D_CTNEA(0)); 22321461365SEric Anholt 22421461365SEric Anholt state->ct1ca = V3D_READ(V3D_CTNCA(1)); 22521461365SEric Anholt state->ct1ea = V3D_READ(V3D_CTNEA(1)); 22621461365SEric Anholt 22721461365SEric Anholt state->ct0cs = V3D_READ(V3D_CTNCS(0)); 22821461365SEric Anholt state->ct1cs = V3D_READ(V3D_CTNCS(1)); 22921461365SEric Anholt 23021461365SEric Anholt state->ct0ra0 = V3D_READ(V3D_CT00RA0); 23121461365SEric Anholt state->ct1ra0 = V3D_READ(V3D_CT01RA0); 23221461365SEric Anholt 23321461365SEric Anholt state->bpca = V3D_READ(V3D_BPCA); 23421461365SEric Anholt state->bpcs = V3D_READ(V3D_BPCS); 23521461365SEric Anholt state->bpoa = V3D_READ(V3D_BPOA); 23621461365SEric Anholt state->bpos = V3D_READ(V3D_BPOS); 23721461365SEric Anholt 23821461365SEric Anholt state->vpmbase = V3D_READ(V3D_VPMBASE); 23921461365SEric Anholt 24021461365SEric Anholt state->dbge = V3D_READ(V3D_DBGE); 24121461365SEric Anholt state->fdbgo = V3D_READ(V3D_FDBGO); 24221461365SEric Anholt state->fdbgb = V3D_READ(V3D_FDBGB); 24321461365SEric Anholt state->fdbgr = V3D_READ(V3D_FDBGR); 24421461365SEric Anholt state->fdbgs = V3D_READ(V3D_FDBGS); 24521461365SEric Anholt state->errstat = V3D_READ(V3D_ERRSTAT); 24621461365SEric Anholt 247b9f19259SBoris Brezillon /* We need to turn purgeable BOs into unpurgeable ones so that 248b9f19259SBoris Brezillon * userspace has a chance to dump the hang state before the kernel 249b9f19259SBoris Brezillon * decides to purge those BOs. 250b9f19259SBoris Brezillon * Note that BO consistency at dump time cannot be guaranteed. For 251b9f19259SBoris Brezillon * example, if the owner of these BOs decides to re-use them or mark 252b9f19259SBoris Brezillon * them purgeable again there's nothing we can do to prevent it. 253b9f19259SBoris Brezillon */ 254b9f19259SBoris Brezillon for (i = 0; i < kernel_state->user_state.bo_count; i++) { 255b9f19259SBoris Brezillon struct vc4_bo *bo = to_vc4_bo(kernel_state->bo[i]); 256b9f19259SBoris Brezillon 257b9f19259SBoris Brezillon if (bo->madv == __VC4_MADV_NOTSUPP) 258b9f19259SBoris Brezillon continue; 259b9f19259SBoris Brezillon 260b9f19259SBoris Brezillon mutex_lock(&bo->madv_lock); 261b9f19259SBoris Brezillon if (!WARN_ON(bo->madv == __VC4_MADV_PURGED)) 262b9f19259SBoris Brezillon bo->madv = VC4_MADV_WILLNEED; 263b9f19259SBoris Brezillon refcount_dec(&bo->usecnt); 264b9f19259SBoris Brezillon mutex_unlock(&bo->madv_lock); 265b9f19259SBoris Brezillon } 266b9f19259SBoris Brezillon 26721461365SEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 26821461365SEric Anholt if (vc4->hang_state) { 26921461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 27021461365SEric Anholt vc4_free_hang_state(dev, kernel_state); 27121461365SEric Anholt } else { 27221461365SEric Anholt vc4->hang_state = kernel_state; 27321461365SEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 27421461365SEric Anholt } 27521461365SEric Anholt } 27621461365SEric Anholt 277d5b1a78aSEric Anholt static void 278d5b1a78aSEric Anholt vc4_reset(struct drm_device *dev) 279d5b1a78aSEric Anholt { 280d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 281d5b1a78aSEric Anholt 282d5b1a78aSEric Anholt DRM_INFO("Resetting GPU.\n"); 28336cb6253SEric Anholt 28436cb6253SEric Anholt mutex_lock(&vc4->power_lock); 28536cb6253SEric Anholt if (vc4->power_refcount) { 28636cb6253SEric Anholt /* Power the device off and back on the by dropping the 28736cb6253SEric Anholt * reference on runtime PM. 28836cb6253SEric Anholt */ 28936cb6253SEric Anholt pm_runtime_put_sync_suspend(&vc4->v3d->pdev->dev); 29036cb6253SEric Anholt pm_runtime_get_sync(&vc4->v3d->pdev->dev); 29136cb6253SEric Anholt } 29236cb6253SEric Anholt mutex_unlock(&vc4->power_lock); 293d5b1a78aSEric Anholt 294d5b1a78aSEric Anholt vc4_irq_reset(dev); 295d5b1a78aSEric Anholt 296d5b1a78aSEric Anholt /* Rearm the hangcheck -- another job might have been waiting 297d5b1a78aSEric Anholt * for our hung one to get kicked off, and vc4_irq_reset() 298d5b1a78aSEric Anholt * would have started it. 299d5b1a78aSEric Anholt */ 300d5b1a78aSEric Anholt vc4_queue_hangcheck(dev); 301d5b1a78aSEric Anholt } 302d5b1a78aSEric Anholt 303d5b1a78aSEric Anholt static void 304d5b1a78aSEric Anholt vc4_reset_work(struct work_struct *work) 305d5b1a78aSEric Anholt { 306d5b1a78aSEric Anholt struct vc4_dev *vc4 = 307d5b1a78aSEric Anholt container_of(work, struct vc4_dev, hangcheck.reset_work); 308d5b1a78aSEric Anholt 30921461365SEric Anholt vc4_save_hang_state(vc4->dev); 31021461365SEric Anholt 311d5b1a78aSEric Anholt vc4_reset(vc4->dev); 312d5b1a78aSEric Anholt } 313d5b1a78aSEric Anholt 314d5b1a78aSEric Anholt static void 3150078730fSKees Cook vc4_hangcheck_elapsed(struct timer_list *t) 316d5b1a78aSEric Anholt { 3170078730fSKees Cook struct vc4_dev *vc4 = from_timer(vc4, t, hangcheck.timer); 3180078730fSKees Cook struct drm_device *dev = vc4->dev; 319d5b1a78aSEric Anholt uint32_t ct0ca, ct1ca; 320c4ce60dcSEric Anholt unsigned long irqflags; 321ca26d28bSVarad Gautam struct vc4_exec_info *bin_exec, *render_exec; 322c4ce60dcSEric Anholt 323c4ce60dcSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 324ca26d28bSVarad Gautam 325ca26d28bSVarad Gautam bin_exec = vc4_first_bin_job(vc4); 326ca26d28bSVarad Gautam render_exec = vc4_first_render_job(vc4); 327d5b1a78aSEric Anholt 328d5b1a78aSEric Anholt /* If idle, we can stop watching for hangs. */ 329ca26d28bSVarad Gautam if (!bin_exec && !render_exec) { 330c4ce60dcSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 331d5b1a78aSEric Anholt return; 332c4ce60dcSEric Anholt } 333d5b1a78aSEric Anholt 334d5b1a78aSEric Anholt ct0ca = V3D_READ(V3D_CTNCA(0)); 335d5b1a78aSEric Anholt ct1ca = V3D_READ(V3D_CTNCA(1)); 336d5b1a78aSEric Anholt 337d5b1a78aSEric Anholt /* If we've made any progress in execution, rearm the timer 338d5b1a78aSEric Anholt * and wait. 339d5b1a78aSEric Anholt */ 340ca26d28bSVarad Gautam if ((bin_exec && ct0ca != bin_exec->last_ct0ca) || 341ca26d28bSVarad Gautam (render_exec && ct1ca != render_exec->last_ct1ca)) { 342ca26d28bSVarad Gautam if (bin_exec) 343ca26d28bSVarad Gautam bin_exec->last_ct0ca = ct0ca; 344ca26d28bSVarad Gautam if (render_exec) 345ca26d28bSVarad Gautam render_exec->last_ct1ca = ct1ca; 346c4ce60dcSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 347d5b1a78aSEric Anholt vc4_queue_hangcheck(dev); 348d5b1a78aSEric Anholt return; 349d5b1a78aSEric Anholt } 350d5b1a78aSEric Anholt 351c4ce60dcSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 352c4ce60dcSEric Anholt 353d5b1a78aSEric Anholt /* We've gone too long with no progress, reset. This has to 354d5b1a78aSEric Anholt * be done from a work struct, since resetting can sleep and 355d5b1a78aSEric Anholt * this timer hook isn't allowed to. 356d5b1a78aSEric Anholt */ 357d5b1a78aSEric Anholt schedule_work(&vc4->hangcheck.reset_work); 358d5b1a78aSEric Anholt } 359d5b1a78aSEric Anholt 360d5b1a78aSEric Anholt static void 361d5b1a78aSEric Anholt submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end) 362d5b1a78aSEric Anholt { 363d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 364d5b1a78aSEric Anholt 365d5b1a78aSEric Anholt /* Set the current and end address of the control list. 366d5b1a78aSEric Anholt * Writing the end register is what starts the job. 367d5b1a78aSEric Anholt */ 368d5b1a78aSEric Anholt V3D_WRITE(V3D_CTNCA(thread), start); 369d5b1a78aSEric Anholt V3D_WRITE(V3D_CTNEA(thread), end); 370d5b1a78aSEric Anholt } 371d5b1a78aSEric Anholt 372d5b1a78aSEric Anholt int 373d5b1a78aSEric Anholt vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns, 374d5b1a78aSEric Anholt bool interruptible) 375d5b1a78aSEric Anholt { 376d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 377d5b1a78aSEric Anholt int ret = 0; 378d5b1a78aSEric Anholt unsigned long timeout_expire; 379d5b1a78aSEric Anholt DEFINE_WAIT(wait); 380d5b1a78aSEric Anholt 381d5b1a78aSEric Anholt if (vc4->finished_seqno >= seqno) 382d5b1a78aSEric Anholt return 0; 383d5b1a78aSEric Anholt 384d5b1a78aSEric Anholt if (timeout_ns == 0) 385d5b1a78aSEric Anholt return -ETIME; 386d5b1a78aSEric Anholt 387d5b1a78aSEric Anholt timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns); 388d5b1a78aSEric Anholt 389d5b1a78aSEric Anholt trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns); 390d5b1a78aSEric Anholt for (;;) { 391d5b1a78aSEric Anholt prepare_to_wait(&vc4->job_wait_queue, &wait, 392d5b1a78aSEric Anholt interruptible ? TASK_INTERRUPTIBLE : 393d5b1a78aSEric Anholt TASK_UNINTERRUPTIBLE); 394d5b1a78aSEric Anholt 395d5b1a78aSEric Anholt if (interruptible && signal_pending(current)) { 396d5b1a78aSEric Anholt ret = -ERESTARTSYS; 397d5b1a78aSEric Anholt break; 398d5b1a78aSEric Anholt } 399d5b1a78aSEric Anholt 400d5b1a78aSEric Anholt if (vc4->finished_seqno >= seqno) 401d5b1a78aSEric Anholt break; 402d5b1a78aSEric Anholt 403d5b1a78aSEric Anholt if (timeout_ns != ~0ull) { 404d5b1a78aSEric Anholt if (time_after_eq(jiffies, timeout_expire)) { 405d5b1a78aSEric Anholt ret = -ETIME; 406d5b1a78aSEric Anholt break; 407d5b1a78aSEric Anholt } 408d5b1a78aSEric Anholt schedule_timeout(timeout_expire - jiffies); 409d5b1a78aSEric Anholt } else { 410d5b1a78aSEric Anholt schedule(); 411d5b1a78aSEric Anholt } 412d5b1a78aSEric Anholt } 413d5b1a78aSEric Anholt 414d5b1a78aSEric Anholt finish_wait(&vc4->job_wait_queue, &wait); 415d5b1a78aSEric Anholt trace_vc4_wait_for_seqno_end(dev, seqno); 416d5b1a78aSEric Anholt 41713cf8909SEric Anholt return ret; 418d5b1a78aSEric Anholt } 419d5b1a78aSEric Anholt 420d5b1a78aSEric Anholt static void 421d5b1a78aSEric Anholt vc4_flush_caches(struct drm_device *dev) 422d5b1a78aSEric Anholt { 423d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 424d5b1a78aSEric Anholt 425d5b1a78aSEric Anholt /* Flush the GPU L2 caches. These caches sit on top of system 426d5b1a78aSEric Anholt * L3 (the 128kb or so shared with the CPU), and are 427d5b1a78aSEric Anholt * non-allocating in the L3. 428d5b1a78aSEric Anholt */ 429d5b1a78aSEric Anholt V3D_WRITE(V3D_L2CACTL, 430d5b1a78aSEric Anholt V3D_L2CACTL_L2CCLR); 431d5b1a78aSEric Anholt 432d5b1a78aSEric Anholt V3D_WRITE(V3D_SLCACTL, 433d5b1a78aSEric Anholt VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) | 434d5b1a78aSEric Anholt VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) | 435d5b1a78aSEric Anholt VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) | 436d5b1a78aSEric Anholt VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC)); 437d5b1a78aSEric Anholt } 438d5b1a78aSEric Anholt 439d5b1a78aSEric Anholt /* Sets the registers for the next job to be actually be executed in 440d5b1a78aSEric Anholt * the hardware. 441d5b1a78aSEric Anholt * 442d5b1a78aSEric Anholt * The job_lock should be held during this. 443d5b1a78aSEric Anholt */ 444d5b1a78aSEric Anholt void 445ca26d28bSVarad Gautam vc4_submit_next_bin_job(struct drm_device *dev) 446d5b1a78aSEric Anholt { 447d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 448ca26d28bSVarad Gautam struct vc4_exec_info *exec; 449d5b1a78aSEric Anholt 450ca26d28bSVarad Gautam again: 451ca26d28bSVarad Gautam exec = vc4_first_bin_job(vc4); 452d5b1a78aSEric Anholt if (!exec) 453d5b1a78aSEric Anholt return; 454d5b1a78aSEric Anholt 455d5b1a78aSEric Anholt vc4_flush_caches(dev); 456d5b1a78aSEric Anholt 45765101d8cSBoris Brezillon /* Only start the perfmon if it was not already started by a previous 45865101d8cSBoris Brezillon * job. 45965101d8cSBoris Brezillon */ 46065101d8cSBoris Brezillon if (exec->perfmon && vc4->active_perfmon != exec->perfmon) 46165101d8cSBoris Brezillon vc4_perfmon_start(vc4, exec->perfmon); 46265101d8cSBoris Brezillon 463ca26d28bSVarad Gautam /* Either put the job in the binner if it uses the binner, or 464ca26d28bSVarad Gautam * immediately move it to the to-be-rendered queue. 465ca26d28bSVarad Gautam */ 466ca26d28bSVarad Gautam if (exec->ct0ca != exec->ct0ea) { 467d5b1a78aSEric Anholt submit_cl(dev, 0, exec->ct0ca, exec->ct0ea); 468ca26d28bSVarad Gautam } else { 46965101d8cSBoris Brezillon struct vc4_exec_info *next; 47065101d8cSBoris Brezillon 471ca26d28bSVarad Gautam vc4_move_job_to_render(dev, exec); 47265101d8cSBoris Brezillon next = vc4_first_bin_job(vc4); 47365101d8cSBoris Brezillon 47465101d8cSBoris Brezillon /* We can't start the next bin job if the previous job had a 47565101d8cSBoris Brezillon * different perfmon instance attached to it. The same goes 47665101d8cSBoris Brezillon * if one of them had a perfmon attached to it and the other 47765101d8cSBoris Brezillon * one doesn't. 47865101d8cSBoris Brezillon */ 47965101d8cSBoris Brezillon if (next && next->perfmon == exec->perfmon) 480ca26d28bSVarad Gautam goto again; 481ca26d28bSVarad Gautam } 482ca26d28bSVarad Gautam } 483ca26d28bSVarad Gautam 484ca26d28bSVarad Gautam void 485ca26d28bSVarad Gautam vc4_submit_next_render_job(struct drm_device *dev) 486ca26d28bSVarad Gautam { 487ca26d28bSVarad Gautam struct vc4_dev *vc4 = to_vc4_dev(dev); 488ca26d28bSVarad Gautam struct vc4_exec_info *exec = vc4_first_render_job(vc4); 489ca26d28bSVarad Gautam 490ca26d28bSVarad Gautam if (!exec) 491ca26d28bSVarad Gautam return; 492ca26d28bSVarad Gautam 493d5b1a78aSEric Anholt submit_cl(dev, 1, exec->ct1ca, exec->ct1ea); 494d5b1a78aSEric Anholt } 495d5b1a78aSEric Anholt 496ca26d28bSVarad Gautam void 497ca26d28bSVarad Gautam vc4_move_job_to_render(struct drm_device *dev, struct vc4_exec_info *exec) 498ca26d28bSVarad Gautam { 499ca26d28bSVarad Gautam struct vc4_dev *vc4 = to_vc4_dev(dev); 500ca26d28bSVarad Gautam bool was_empty = list_empty(&vc4->render_job_list); 501ca26d28bSVarad Gautam 502ca26d28bSVarad Gautam list_move_tail(&exec->head, &vc4->render_job_list); 503ca26d28bSVarad Gautam if (was_empty) 504ca26d28bSVarad Gautam vc4_submit_next_render_job(dev); 505ca26d28bSVarad Gautam } 506ca26d28bSVarad Gautam 507d5b1a78aSEric Anholt static void 508d5b1a78aSEric Anholt vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno) 509d5b1a78aSEric Anholt { 510d5b1a78aSEric Anholt struct vc4_bo *bo; 511d5b1a78aSEric Anholt unsigned i; 512d5b1a78aSEric Anholt 513d5b1a78aSEric Anholt for (i = 0; i < exec->bo_count; i++) { 514d5b1a78aSEric Anholt bo = to_vc4_bo(&exec->bo[i]->base); 515d5b1a78aSEric Anholt bo->seqno = seqno; 516cdec4d36SEric Anholt 517cdec4d36SEric Anholt reservation_object_add_shared_fence(bo->resv, exec->fence); 518d5b1a78aSEric Anholt } 519d5b1a78aSEric Anholt 520d5b1a78aSEric Anholt list_for_each_entry(bo, &exec->unref_list, unref_head) { 521d5b1a78aSEric Anholt bo->seqno = seqno; 522d5b1a78aSEric Anholt } 5237edabee0SEric Anholt 5247edabee0SEric Anholt for (i = 0; i < exec->rcl_write_bo_count; i++) { 5257edabee0SEric Anholt bo = to_vc4_bo(&exec->rcl_write_bo[i]->base); 5267edabee0SEric Anholt bo->write_seqno = seqno; 527cdec4d36SEric Anholt 528cdec4d36SEric Anholt reservation_object_add_excl_fence(bo->resv, exec->fence); 5297edabee0SEric Anholt } 530d5b1a78aSEric Anholt } 531d5b1a78aSEric Anholt 532cdec4d36SEric Anholt static void 533cdec4d36SEric Anholt vc4_unlock_bo_reservations(struct drm_device *dev, 534cdec4d36SEric Anholt struct vc4_exec_info *exec, 535cdec4d36SEric Anholt struct ww_acquire_ctx *acquire_ctx) 536cdec4d36SEric Anholt { 537cdec4d36SEric Anholt int i; 538cdec4d36SEric Anholt 539cdec4d36SEric Anholt for (i = 0; i < exec->bo_count; i++) { 540cdec4d36SEric Anholt struct vc4_bo *bo = to_vc4_bo(&exec->bo[i]->base); 541cdec4d36SEric Anholt 542cdec4d36SEric Anholt ww_mutex_unlock(&bo->resv->lock); 543cdec4d36SEric Anholt } 544cdec4d36SEric Anholt 545cdec4d36SEric Anholt ww_acquire_fini(acquire_ctx); 546cdec4d36SEric Anholt } 547cdec4d36SEric Anholt 548cdec4d36SEric Anholt /* Takes the reservation lock on all the BOs being referenced, so that 549cdec4d36SEric Anholt * at queue submit time we can update the reservations. 550cdec4d36SEric Anholt * 551cdec4d36SEric Anholt * We don't lock the RCL the tile alloc/state BOs, or overflow memory 552cdec4d36SEric Anholt * (all of which are on exec->unref_list). They're entirely private 553cdec4d36SEric Anholt * to vc4, so we don't attach dma-buf fences to them. 554cdec4d36SEric Anholt */ 555cdec4d36SEric Anholt static int 556cdec4d36SEric Anholt vc4_lock_bo_reservations(struct drm_device *dev, 557cdec4d36SEric Anholt struct vc4_exec_info *exec, 558cdec4d36SEric Anholt struct ww_acquire_ctx *acquire_ctx) 559cdec4d36SEric Anholt { 560cdec4d36SEric Anholt int contended_lock = -1; 561cdec4d36SEric Anholt int i, ret; 562cdec4d36SEric Anholt struct vc4_bo *bo; 563cdec4d36SEric Anholt 564cdec4d36SEric Anholt ww_acquire_init(acquire_ctx, &reservation_ww_class); 565cdec4d36SEric Anholt 566cdec4d36SEric Anholt retry: 567cdec4d36SEric Anholt if (contended_lock != -1) { 568cdec4d36SEric Anholt bo = to_vc4_bo(&exec->bo[contended_lock]->base); 569cdec4d36SEric Anholt ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, 570cdec4d36SEric Anholt acquire_ctx); 571cdec4d36SEric Anholt if (ret) { 572cdec4d36SEric Anholt ww_acquire_done(acquire_ctx); 573cdec4d36SEric Anholt return ret; 574cdec4d36SEric Anholt } 575cdec4d36SEric Anholt } 576cdec4d36SEric Anholt 577cdec4d36SEric Anholt for (i = 0; i < exec->bo_count; i++) { 578cdec4d36SEric Anholt if (i == contended_lock) 579cdec4d36SEric Anholt continue; 580cdec4d36SEric Anholt 581cdec4d36SEric Anholt bo = to_vc4_bo(&exec->bo[i]->base); 582cdec4d36SEric Anholt 583cdec4d36SEric Anholt ret = ww_mutex_lock_interruptible(&bo->resv->lock, acquire_ctx); 584cdec4d36SEric Anholt if (ret) { 585cdec4d36SEric Anholt int j; 586cdec4d36SEric Anholt 587cdec4d36SEric Anholt for (j = 0; j < i; j++) { 588cdec4d36SEric Anholt bo = to_vc4_bo(&exec->bo[j]->base); 589cdec4d36SEric Anholt ww_mutex_unlock(&bo->resv->lock); 590cdec4d36SEric Anholt } 591cdec4d36SEric Anholt 592cdec4d36SEric Anholt if (contended_lock != -1 && contended_lock >= i) { 593cdec4d36SEric Anholt bo = to_vc4_bo(&exec->bo[contended_lock]->base); 594cdec4d36SEric Anholt 595cdec4d36SEric Anholt ww_mutex_unlock(&bo->resv->lock); 596cdec4d36SEric Anholt } 597cdec4d36SEric Anholt 598cdec4d36SEric Anholt if (ret == -EDEADLK) { 599cdec4d36SEric Anholt contended_lock = i; 600cdec4d36SEric Anholt goto retry; 601cdec4d36SEric Anholt } 602cdec4d36SEric Anholt 603cdec4d36SEric Anholt ww_acquire_done(acquire_ctx); 604cdec4d36SEric Anholt return ret; 605cdec4d36SEric Anholt } 606cdec4d36SEric Anholt } 607cdec4d36SEric Anholt 608cdec4d36SEric Anholt ww_acquire_done(acquire_ctx); 609cdec4d36SEric Anholt 610cdec4d36SEric Anholt /* Reserve space for our shared (read-only) fence references, 611cdec4d36SEric Anholt * before we commit the CL to the hardware. 612cdec4d36SEric Anholt */ 613cdec4d36SEric Anholt for (i = 0; i < exec->bo_count; i++) { 614cdec4d36SEric Anholt bo = to_vc4_bo(&exec->bo[i]->base); 615cdec4d36SEric Anholt 616cdec4d36SEric Anholt ret = reservation_object_reserve_shared(bo->resv); 617cdec4d36SEric Anholt if (ret) { 618cdec4d36SEric Anholt vc4_unlock_bo_reservations(dev, exec, acquire_ctx); 619cdec4d36SEric Anholt return ret; 620cdec4d36SEric Anholt } 621cdec4d36SEric Anholt } 622cdec4d36SEric Anholt 623cdec4d36SEric Anholt return 0; 624cdec4d36SEric Anholt } 625cdec4d36SEric Anholt 626d5b1a78aSEric Anholt /* Queues a struct vc4_exec_info for execution. If no job is 627d5b1a78aSEric Anholt * currently executing, then submits it. 628d5b1a78aSEric Anholt * 629d5b1a78aSEric Anholt * Unlike most GPUs, our hardware only handles one command list at a 630d5b1a78aSEric Anholt * time. To queue multiple jobs at once, we'd need to edit the 631d5b1a78aSEric Anholt * previous command list to have a jump to the new one at the end, and 632d5b1a78aSEric Anholt * then bump the end address. That's a change for a later date, 633d5b1a78aSEric Anholt * though. 634d5b1a78aSEric Anholt */ 635cdec4d36SEric Anholt static int 636cdec4d36SEric Anholt vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec, 637cdec4d36SEric Anholt struct ww_acquire_ctx *acquire_ctx) 638d5b1a78aSEric Anholt { 639d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 64065101d8cSBoris Brezillon struct vc4_exec_info *renderjob; 641d5b1a78aSEric Anholt uint64_t seqno; 642d5b1a78aSEric Anholt unsigned long irqflags; 643cdec4d36SEric Anholt struct vc4_fence *fence; 644cdec4d36SEric Anholt 645cdec4d36SEric Anholt fence = kzalloc(sizeof(*fence), GFP_KERNEL); 646cdec4d36SEric Anholt if (!fence) 647cdec4d36SEric Anholt return -ENOMEM; 648cdec4d36SEric Anholt fence->dev = dev; 649d5b1a78aSEric Anholt 650d5b1a78aSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 651d5b1a78aSEric Anholt 652d5b1a78aSEric Anholt seqno = ++vc4->emit_seqno; 653d5b1a78aSEric Anholt exec->seqno = seqno; 654cdec4d36SEric Anholt 655cdec4d36SEric Anholt dma_fence_init(&fence->base, &vc4_fence_ops, &vc4->job_lock, 656cdec4d36SEric Anholt vc4->dma_fence_context, exec->seqno); 657cdec4d36SEric Anholt fence->seqno = exec->seqno; 658cdec4d36SEric Anholt exec->fence = &fence->base; 659cdec4d36SEric Anholt 660d5b1a78aSEric Anholt vc4_update_bo_seqnos(exec, seqno); 661d5b1a78aSEric Anholt 662cdec4d36SEric Anholt vc4_unlock_bo_reservations(dev, exec, acquire_ctx); 663cdec4d36SEric Anholt 664ca26d28bSVarad Gautam list_add_tail(&exec->head, &vc4->bin_job_list); 665d5b1a78aSEric Anholt 66665101d8cSBoris Brezillon /* If no bin job was executing and if the render job (if any) has the 66765101d8cSBoris Brezillon * same perfmon as our job attached to it (or if both jobs don't have 66865101d8cSBoris Brezillon * perfmon activated), then kick ours off. Otherwise, it'll get 66965101d8cSBoris Brezillon * started when the previous job's flush/render done interrupt occurs. 670d5b1a78aSEric Anholt */ 67165101d8cSBoris Brezillon renderjob = vc4_first_render_job(vc4); 67265101d8cSBoris Brezillon if (vc4_first_bin_job(vc4) == exec && 67365101d8cSBoris Brezillon (!renderjob || renderjob->perfmon == exec->perfmon)) { 674ca26d28bSVarad Gautam vc4_submit_next_bin_job(dev); 675d5b1a78aSEric Anholt vc4_queue_hangcheck(dev); 676d5b1a78aSEric Anholt } 677d5b1a78aSEric Anholt 678d5b1a78aSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 679cdec4d36SEric Anholt 680cdec4d36SEric Anholt return 0; 681d5b1a78aSEric Anholt } 682d5b1a78aSEric Anholt 683d5b1a78aSEric Anholt /** 68472f793f1SEric Anholt * vc4_cl_lookup_bos() - Sets up exec->bo[] with the GEM objects 68572f793f1SEric Anholt * referenced by the job. 68672f793f1SEric Anholt * @dev: DRM device 68772f793f1SEric Anholt * @file_priv: DRM file for this fd 68872f793f1SEric Anholt * @exec: V3D job being set up 68972f793f1SEric Anholt * 69072f793f1SEric Anholt * The command validator needs to reference BOs by their index within 69172f793f1SEric Anholt * the submitted job's BO list. This does the validation of the job's 69272f793f1SEric Anholt * BO list and reference counting for the lifetime of the job. 693d5b1a78aSEric Anholt */ 694d5b1a78aSEric Anholt static int 695d5b1a78aSEric Anholt vc4_cl_lookup_bos(struct drm_device *dev, 696d5b1a78aSEric Anholt struct drm_file *file_priv, 697d5b1a78aSEric Anholt struct vc4_exec_info *exec) 698d5b1a78aSEric Anholt { 699d5b1a78aSEric Anholt struct drm_vc4_submit_cl *args = exec->args; 700d5b1a78aSEric Anholt uint32_t *handles; 701d5b1a78aSEric Anholt int ret = 0; 702d5b1a78aSEric Anholt int i; 703d5b1a78aSEric Anholt 704d5b1a78aSEric Anholt exec->bo_count = args->bo_handle_count; 705d5b1a78aSEric Anholt 706d5b1a78aSEric Anholt if (!exec->bo_count) { 707d5b1a78aSEric Anholt /* See comment on bo_index for why we have to check 708d5b1a78aSEric Anholt * this. 709d5b1a78aSEric Anholt */ 710fb95992aSEric Anholt DRM_DEBUG("Rendering requires BOs to validate\n"); 711d5b1a78aSEric Anholt return -EINVAL; 712d5b1a78aSEric Anholt } 713d5b1a78aSEric Anholt 7142098105eSMichal Hocko exec->bo = kvmalloc_array(exec->bo_count, 7152098105eSMichal Hocko sizeof(struct drm_gem_cma_object *), 7162098105eSMichal Hocko GFP_KERNEL | __GFP_ZERO); 717d5b1a78aSEric Anholt if (!exec->bo) { 718d5b1a78aSEric Anholt DRM_ERROR("Failed to allocate validated BO pointers\n"); 719d5b1a78aSEric Anholt return -ENOMEM; 720d5b1a78aSEric Anholt } 721d5b1a78aSEric Anholt 7222098105eSMichal Hocko handles = kvmalloc_array(exec->bo_count, sizeof(uint32_t), GFP_KERNEL); 723d5b1a78aSEric Anholt if (!handles) { 724b2cdeb19SDan Carpenter ret = -ENOMEM; 725d5b1a78aSEric Anholt DRM_ERROR("Failed to allocate incoming GEM handles\n"); 726d5b1a78aSEric Anholt goto fail; 727d5b1a78aSEric Anholt } 728d5b1a78aSEric Anholt 72995d7cbcbSEric Anholt if (copy_from_user(handles, u64_to_user_ptr(args->bo_handles), 730b2cdeb19SDan Carpenter exec->bo_count * sizeof(uint32_t))) { 731b2cdeb19SDan Carpenter ret = -EFAULT; 732d5b1a78aSEric Anholt DRM_ERROR("Failed to copy in GEM handles\n"); 733d5b1a78aSEric Anholt goto fail; 734d5b1a78aSEric Anholt } 735d5b1a78aSEric Anholt 736d5b1a78aSEric Anholt spin_lock(&file_priv->table_lock); 737d5b1a78aSEric Anholt for (i = 0; i < exec->bo_count; i++) { 738d5b1a78aSEric Anholt struct drm_gem_object *bo = idr_find(&file_priv->object_idr, 739d5b1a78aSEric Anholt handles[i]); 740d5b1a78aSEric Anholt if (!bo) { 741fb95992aSEric Anholt DRM_DEBUG("Failed to look up GEM BO %d: %d\n", 742d5b1a78aSEric Anholt i, handles[i]); 743d5b1a78aSEric Anholt ret = -EINVAL; 744b9f19259SBoris Brezillon break; 745d5b1a78aSEric Anholt } 746b9f19259SBoris Brezillon 7471d5494e9SCihangir Akturk drm_gem_object_get(bo); 748d5b1a78aSEric Anholt exec->bo[i] = (struct drm_gem_cma_object *)bo; 749d5b1a78aSEric Anholt } 750d5b1a78aSEric Anholt spin_unlock(&file_priv->table_lock); 751d5b1a78aSEric Anholt 752b9f19259SBoris Brezillon if (ret) 753b9f19259SBoris Brezillon goto fail_put_bo; 754b9f19259SBoris Brezillon 755b9f19259SBoris Brezillon for (i = 0; i < exec->bo_count; i++) { 756b9f19259SBoris Brezillon ret = vc4_bo_inc_usecnt(to_vc4_bo(&exec->bo[i]->base)); 757b9f19259SBoris Brezillon if (ret) 758b9f19259SBoris Brezillon goto fail_dec_usecnt; 759b9f19259SBoris Brezillon } 760b9f19259SBoris Brezillon 761b9f19259SBoris Brezillon kvfree(handles); 762b9f19259SBoris Brezillon return 0; 763b9f19259SBoris Brezillon 764b9f19259SBoris Brezillon fail_dec_usecnt: 765b9f19259SBoris Brezillon /* Decrease usecnt on acquired objects. 766b9f19259SBoris Brezillon * We cannot rely on vc4_complete_exec() to release resources here, 767b9f19259SBoris Brezillon * because vc4_complete_exec() has no information about which BO has 768b9f19259SBoris Brezillon * had its ->usecnt incremented. 769b9f19259SBoris Brezillon * To make things easier we just free everything explicitly and set 770b9f19259SBoris Brezillon * exec->bo to NULL so that vc4_complete_exec() skips the 'BO release' 771b9f19259SBoris Brezillon * step. 772b9f19259SBoris Brezillon */ 773b9f19259SBoris Brezillon for (i-- ; i >= 0; i--) 774b9f19259SBoris Brezillon vc4_bo_dec_usecnt(to_vc4_bo(&exec->bo[i]->base)); 775b9f19259SBoris Brezillon 776b9f19259SBoris Brezillon fail_put_bo: 777b9f19259SBoris Brezillon /* Release any reference to acquired objects. */ 778b9f19259SBoris Brezillon for (i = 0; i < exec->bo_count && exec->bo[i]; i++) 779b9f19259SBoris Brezillon drm_gem_object_put_unlocked(&exec->bo[i]->base); 780b9f19259SBoris Brezillon 781d5b1a78aSEric Anholt fail: 7822098105eSMichal Hocko kvfree(handles); 783b9f19259SBoris Brezillon kvfree(exec->bo); 784b9f19259SBoris Brezillon exec->bo = NULL; 785552416c1SEric Anholt return ret; 786d5b1a78aSEric Anholt } 787d5b1a78aSEric Anholt 788d5b1a78aSEric Anholt static int 789d5b1a78aSEric Anholt vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec) 790d5b1a78aSEric Anholt { 791d5b1a78aSEric Anholt struct drm_vc4_submit_cl *args = exec->args; 792d5b1a78aSEric Anholt void *temp = NULL; 793d5b1a78aSEric Anholt void *bin; 794d5b1a78aSEric Anholt int ret = 0; 795d5b1a78aSEric Anholt uint32_t bin_offset = 0; 796d5b1a78aSEric Anholt uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size, 797d5b1a78aSEric Anholt 16); 798d5b1a78aSEric Anholt uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size; 799d5b1a78aSEric Anholt uint32_t exec_size = uniforms_offset + args->uniforms_size; 800d5b1a78aSEric Anholt uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) * 801d5b1a78aSEric Anholt args->shader_rec_count); 802d5b1a78aSEric Anholt struct vc4_bo *bo; 803d5b1a78aSEric Anholt 8040f2ff82eSEric Anholt if (shader_rec_offset < args->bin_cl_size || 8050f2ff82eSEric Anholt uniforms_offset < shader_rec_offset || 806d5b1a78aSEric Anholt exec_size < uniforms_offset || 807d5b1a78aSEric Anholt args->shader_rec_count >= (UINT_MAX / 808d5b1a78aSEric Anholt sizeof(struct vc4_shader_state)) || 809d5b1a78aSEric Anholt temp_size < exec_size) { 810fb95992aSEric Anholt DRM_DEBUG("overflow in exec arguments\n"); 8116b8ac638SEric Anholt ret = -EINVAL; 812d5b1a78aSEric Anholt goto fail; 813d5b1a78aSEric Anholt } 814d5b1a78aSEric Anholt 815d5b1a78aSEric Anholt /* Allocate space where we'll store the copied in user command lists 816d5b1a78aSEric Anholt * and shader records. 817d5b1a78aSEric Anholt * 818d5b1a78aSEric Anholt * We don't just copy directly into the BOs because we need to 819d5b1a78aSEric Anholt * read the contents back for validation, and I think the 820d5b1a78aSEric Anholt * bo->vaddr is uncached access. 821d5b1a78aSEric Anholt */ 8222098105eSMichal Hocko temp = kvmalloc_array(temp_size, 1, GFP_KERNEL); 823d5b1a78aSEric Anholt if (!temp) { 824d5b1a78aSEric Anholt DRM_ERROR("Failed to allocate storage for copying " 825d5b1a78aSEric Anholt "in bin/render CLs.\n"); 826d5b1a78aSEric Anholt ret = -ENOMEM; 827d5b1a78aSEric Anholt goto fail; 828d5b1a78aSEric Anholt } 829d5b1a78aSEric Anholt bin = temp + bin_offset; 830d5b1a78aSEric Anholt exec->shader_rec_u = temp + shader_rec_offset; 831d5b1a78aSEric Anholt exec->uniforms_u = temp + uniforms_offset; 832d5b1a78aSEric Anholt exec->shader_state = temp + exec_size; 833d5b1a78aSEric Anholt exec->shader_state_size = args->shader_rec_count; 834d5b1a78aSEric Anholt 83565c4777dSDan Carpenter if (copy_from_user(bin, 83695d7cbcbSEric Anholt u64_to_user_ptr(args->bin_cl), 83765c4777dSDan Carpenter args->bin_cl_size)) { 83865c4777dSDan Carpenter ret = -EFAULT; 839d5b1a78aSEric Anholt goto fail; 840d5b1a78aSEric Anholt } 841d5b1a78aSEric Anholt 84265c4777dSDan Carpenter if (copy_from_user(exec->shader_rec_u, 84395d7cbcbSEric Anholt u64_to_user_ptr(args->shader_rec), 84465c4777dSDan Carpenter args->shader_rec_size)) { 84565c4777dSDan Carpenter ret = -EFAULT; 846d5b1a78aSEric Anholt goto fail; 847d5b1a78aSEric Anholt } 848d5b1a78aSEric Anholt 84965c4777dSDan Carpenter if (copy_from_user(exec->uniforms_u, 85095d7cbcbSEric Anholt u64_to_user_ptr(args->uniforms), 85165c4777dSDan Carpenter args->uniforms_size)) { 85265c4777dSDan Carpenter ret = -EFAULT; 853d5b1a78aSEric Anholt goto fail; 854d5b1a78aSEric Anholt } 855d5b1a78aSEric Anholt 856f3099462SEric Anholt bo = vc4_bo_create(dev, exec_size, true, VC4_BO_TYPE_BCL); 8572c68f1fcSEric Anholt if (IS_ERR(bo)) { 858d5b1a78aSEric Anholt DRM_ERROR("Couldn't allocate BO for binning\n"); 8592c68f1fcSEric Anholt ret = PTR_ERR(bo); 860d5b1a78aSEric Anholt goto fail; 861d5b1a78aSEric Anholt } 862d5b1a78aSEric Anholt exec->exec_bo = &bo->base; 863d5b1a78aSEric Anholt 864d5b1a78aSEric Anholt list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head, 865d5b1a78aSEric Anholt &exec->unref_list); 866d5b1a78aSEric Anholt 867d5b1a78aSEric Anholt exec->ct0ca = exec->exec_bo->paddr + bin_offset; 868d5b1a78aSEric Anholt 869d5b1a78aSEric Anholt exec->bin_u = bin; 870d5b1a78aSEric Anholt 871d5b1a78aSEric Anholt exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset; 872d5b1a78aSEric Anholt exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset; 873d5b1a78aSEric Anholt exec->shader_rec_size = args->shader_rec_size; 874d5b1a78aSEric Anholt 875d5b1a78aSEric Anholt exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset; 876d5b1a78aSEric Anholt exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset; 877d5b1a78aSEric Anholt exec->uniforms_size = args->uniforms_size; 878d5b1a78aSEric Anholt 879d5b1a78aSEric Anholt ret = vc4_validate_bin_cl(dev, 880d5b1a78aSEric Anholt exec->exec_bo->vaddr + bin_offset, 881d5b1a78aSEric Anholt bin, 882d5b1a78aSEric Anholt exec); 883d5b1a78aSEric Anholt if (ret) 884d5b1a78aSEric Anholt goto fail; 885d5b1a78aSEric Anholt 886d5b1a78aSEric Anholt ret = vc4_validate_shader_recs(dev, exec); 8877edabee0SEric Anholt if (ret) 8887edabee0SEric Anholt goto fail; 8897edabee0SEric Anholt 8907edabee0SEric Anholt /* Block waiting on any previous rendering into the CS's VBO, 8917edabee0SEric Anholt * IB, or textures, so that pixels are actually written by the 8927edabee0SEric Anholt * time we try to read them. 8937edabee0SEric Anholt */ 8947edabee0SEric Anholt ret = vc4_wait_for_seqno(dev, exec->bin_dep_seqno, ~0ull, true); 895d5b1a78aSEric Anholt 896d5b1a78aSEric Anholt fail: 8972098105eSMichal Hocko kvfree(temp); 898d5b1a78aSEric Anholt return ret; 899d5b1a78aSEric Anholt } 900d5b1a78aSEric Anholt 901d5b1a78aSEric Anholt static void 902d5b1a78aSEric Anholt vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec) 903d5b1a78aSEric Anholt { 904001bdb55SEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 905553c942fSEric Anholt unsigned long irqflags; 906d5b1a78aSEric Anholt unsigned i; 907d5b1a78aSEric Anholt 908cdec4d36SEric Anholt /* If we got force-completed because of GPU reset rather than 909cdec4d36SEric Anholt * through our IRQ handler, signal the fence now. 910cdec4d36SEric Anholt */ 911babc8110SStefan Schake if (exec->fence) { 912cdec4d36SEric Anholt dma_fence_signal(exec->fence); 913babc8110SStefan Schake dma_fence_put(exec->fence); 914babc8110SStefan Schake } 915cdec4d36SEric Anholt 916d5b1a78aSEric Anholt if (exec->bo) { 917b9f19259SBoris Brezillon for (i = 0; i < exec->bo_count; i++) { 918b9f19259SBoris Brezillon struct vc4_bo *bo = to_vc4_bo(&exec->bo[i]->base); 919b9f19259SBoris Brezillon 920b9f19259SBoris Brezillon vc4_bo_dec_usecnt(bo); 9211d5494e9SCihangir Akturk drm_gem_object_put_unlocked(&exec->bo[i]->base); 922b9f19259SBoris Brezillon } 9232098105eSMichal Hocko kvfree(exec->bo); 924d5b1a78aSEric Anholt } 925d5b1a78aSEric Anholt 926d5b1a78aSEric Anholt while (!list_empty(&exec->unref_list)) { 927d5b1a78aSEric Anholt struct vc4_bo *bo = list_first_entry(&exec->unref_list, 928d5b1a78aSEric Anholt struct vc4_bo, unref_head); 929d5b1a78aSEric Anholt list_del(&bo->unref_head); 9301d5494e9SCihangir Akturk drm_gem_object_put_unlocked(&bo->base.base); 931d5b1a78aSEric Anholt } 932d5b1a78aSEric Anholt 933553c942fSEric Anholt /* Free up the allocation of any bin slots we used. */ 934553c942fSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 935553c942fSEric Anholt vc4->bin_alloc_used &= ~exec->bin_slots; 936553c942fSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 937553c942fSEric Anholt 93865101d8cSBoris Brezillon /* Release the reference we had on the perf monitor. */ 93965101d8cSBoris Brezillon vc4_perfmon_put(exec->perfmon); 94065101d8cSBoris Brezillon 94136cb6253SEric Anholt mutex_lock(&vc4->power_lock); 9423a622346SEric Anholt if (--vc4->power_refcount == 0) { 9433a622346SEric Anholt pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev); 9443a622346SEric Anholt pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev); 9453a622346SEric Anholt } 94636cb6253SEric Anholt mutex_unlock(&vc4->power_lock); 947001bdb55SEric Anholt 948d5b1a78aSEric Anholt kfree(exec); 949d5b1a78aSEric Anholt } 950d5b1a78aSEric Anholt 951d5b1a78aSEric Anholt void 952d5b1a78aSEric Anholt vc4_job_handle_completed(struct vc4_dev *vc4) 953d5b1a78aSEric Anholt { 954d5b1a78aSEric Anholt unsigned long irqflags; 955b501baccSEric Anholt struct vc4_seqno_cb *cb, *cb_temp; 956d5b1a78aSEric Anholt 957d5b1a78aSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 958d5b1a78aSEric Anholt while (!list_empty(&vc4->job_done_list)) { 959d5b1a78aSEric Anholt struct vc4_exec_info *exec = 960d5b1a78aSEric Anholt list_first_entry(&vc4->job_done_list, 961d5b1a78aSEric Anholt struct vc4_exec_info, head); 962d5b1a78aSEric Anholt list_del(&exec->head); 963d5b1a78aSEric Anholt 964d5b1a78aSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 965d5b1a78aSEric Anholt vc4_complete_exec(vc4->dev, exec); 966d5b1a78aSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 967d5b1a78aSEric Anholt } 968b501baccSEric Anholt 969b501baccSEric Anholt list_for_each_entry_safe(cb, cb_temp, &vc4->seqno_cb_list, work.entry) { 970b501baccSEric Anholt if (cb->seqno <= vc4->finished_seqno) { 971b501baccSEric Anholt list_del_init(&cb->work.entry); 972b501baccSEric Anholt schedule_work(&cb->work); 973b501baccSEric Anholt } 974b501baccSEric Anholt } 975b501baccSEric Anholt 976d5b1a78aSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 977d5b1a78aSEric Anholt } 978d5b1a78aSEric Anholt 979b501baccSEric Anholt static void vc4_seqno_cb_work(struct work_struct *work) 980b501baccSEric Anholt { 981b501baccSEric Anholt struct vc4_seqno_cb *cb = container_of(work, struct vc4_seqno_cb, work); 982b501baccSEric Anholt 983b501baccSEric Anholt cb->func(cb); 984b501baccSEric Anholt } 985b501baccSEric Anholt 986b501baccSEric Anholt int vc4_queue_seqno_cb(struct drm_device *dev, 987b501baccSEric Anholt struct vc4_seqno_cb *cb, uint64_t seqno, 988b501baccSEric Anholt void (*func)(struct vc4_seqno_cb *cb)) 989b501baccSEric Anholt { 990b501baccSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 991b501baccSEric Anholt int ret = 0; 992b501baccSEric Anholt unsigned long irqflags; 993b501baccSEric Anholt 994b501baccSEric Anholt cb->func = func; 995b501baccSEric Anholt INIT_WORK(&cb->work, vc4_seqno_cb_work); 996b501baccSEric Anholt 997b501baccSEric Anholt spin_lock_irqsave(&vc4->job_lock, irqflags); 998b501baccSEric Anholt if (seqno > vc4->finished_seqno) { 999b501baccSEric Anholt cb->seqno = seqno; 1000b501baccSEric Anholt list_add_tail(&cb->work.entry, &vc4->seqno_cb_list); 1001b501baccSEric Anholt } else { 1002b501baccSEric Anholt schedule_work(&cb->work); 1003b501baccSEric Anholt } 1004b501baccSEric Anholt spin_unlock_irqrestore(&vc4->job_lock, irqflags); 1005b501baccSEric Anholt 1006b501baccSEric Anholt return ret; 1007b501baccSEric Anholt } 1008b501baccSEric Anholt 1009d5b1a78aSEric Anholt /* Scheduled when any job has been completed, this walks the list of 1010d5b1a78aSEric Anholt * jobs that had completed and unrefs their BOs and frees their exec 1011d5b1a78aSEric Anholt * structs. 1012d5b1a78aSEric Anholt */ 1013d5b1a78aSEric Anholt static void 1014d5b1a78aSEric Anholt vc4_job_done_work(struct work_struct *work) 1015d5b1a78aSEric Anholt { 1016d5b1a78aSEric Anholt struct vc4_dev *vc4 = 1017d5b1a78aSEric Anholt container_of(work, struct vc4_dev, job_done_work); 1018d5b1a78aSEric Anholt 1019d5b1a78aSEric Anholt vc4_job_handle_completed(vc4); 1020d5b1a78aSEric Anholt } 1021d5b1a78aSEric Anholt 1022d5b1a78aSEric Anholt static int 1023d5b1a78aSEric Anholt vc4_wait_for_seqno_ioctl_helper(struct drm_device *dev, 1024d5b1a78aSEric Anholt uint64_t seqno, 1025d5b1a78aSEric Anholt uint64_t *timeout_ns) 1026d5b1a78aSEric Anholt { 1027d5b1a78aSEric Anholt unsigned long start = jiffies; 1028d5b1a78aSEric Anholt int ret = vc4_wait_for_seqno(dev, seqno, *timeout_ns, true); 1029d5b1a78aSEric Anholt 1030d5b1a78aSEric Anholt if ((ret == -EINTR || ret == -ERESTARTSYS) && *timeout_ns != ~0ull) { 1031d5b1a78aSEric Anholt uint64_t delta = jiffies_to_nsecs(jiffies - start); 1032d5b1a78aSEric Anholt 1033d5b1a78aSEric Anholt if (*timeout_ns >= delta) 1034d5b1a78aSEric Anholt *timeout_ns -= delta; 1035d5b1a78aSEric Anholt } 1036d5b1a78aSEric Anholt 1037d5b1a78aSEric Anholt return ret; 1038d5b1a78aSEric Anholt } 1039d5b1a78aSEric Anholt 1040d5b1a78aSEric Anholt int 1041d5b1a78aSEric Anholt vc4_wait_seqno_ioctl(struct drm_device *dev, void *data, 1042d5b1a78aSEric Anholt struct drm_file *file_priv) 1043d5b1a78aSEric Anholt { 1044d5b1a78aSEric Anholt struct drm_vc4_wait_seqno *args = data; 1045d5b1a78aSEric Anholt 1046d5b1a78aSEric Anholt return vc4_wait_for_seqno_ioctl_helper(dev, args->seqno, 1047d5b1a78aSEric Anholt &args->timeout_ns); 1048d5b1a78aSEric Anholt } 1049d5b1a78aSEric Anholt 1050d5b1a78aSEric Anholt int 1051d5b1a78aSEric Anholt vc4_wait_bo_ioctl(struct drm_device *dev, void *data, 1052d5b1a78aSEric Anholt struct drm_file *file_priv) 1053d5b1a78aSEric Anholt { 1054d5b1a78aSEric Anholt int ret; 1055d5b1a78aSEric Anholt struct drm_vc4_wait_bo *args = data; 1056d5b1a78aSEric Anholt struct drm_gem_object *gem_obj; 1057d5b1a78aSEric Anholt struct vc4_bo *bo; 1058d5b1a78aSEric Anholt 1059e0015236SEric Anholt if (args->pad != 0) 1060e0015236SEric Anholt return -EINVAL; 1061e0015236SEric Anholt 1062a8ad0bd8SChris Wilson gem_obj = drm_gem_object_lookup(file_priv, args->handle); 1063d5b1a78aSEric Anholt if (!gem_obj) { 1064fb95992aSEric Anholt DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 1065d5b1a78aSEric Anholt return -EINVAL; 1066d5b1a78aSEric Anholt } 1067d5b1a78aSEric Anholt bo = to_vc4_bo(gem_obj); 1068d5b1a78aSEric Anholt 1069d5b1a78aSEric Anholt ret = vc4_wait_for_seqno_ioctl_helper(dev, bo->seqno, 1070d5b1a78aSEric Anholt &args->timeout_ns); 1071d5b1a78aSEric Anholt 10721d5494e9SCihangir Akturk drm_gem_object_put_unlocked(gem_obj); 1073d5b1a78aSEric Anholt return ret; 1074d5b1a78aSEric Anholt } 1075d5b1a78aSEric Anholt 1076d5b1a78aSEric Anholt /** 107772f793f1SEric Anholt * vc4_submit_cl_ioctl() - Submits a job (frame) to the VC4. 107872f793f1SEric Anholt * @dev: DRM device 107972f793f1SEric Anholt * @data: ioctl argument 108072f793f1SEric Anholt * @file_priv: DRM file for this fd 1081d5b1a78aSEric Anholt * 108272f793f1SEric Anholt * This is the main entrypoint for userspace to submit a 3D frame to 108372f793f1SEric Anholt * the GPU. Userspace provides the binner command list (if 108472f793f1SEric Anholt * applicable), and the kernel sets up the render command list to draw 108572f793f1SEric Anholt * to the framebuffer described in the ioctl, using the command lists 108672f793f1SEric Anholt * that the 3D engine's binner will produce. 1087d5b1a78aSEric Anholt */ 1088d5b1a78aSEric Anholt int 1089d5b1a78aSEric Anholt vc4_submit_cl_ioctl(struct drm_device *dev, void *data, 1090d5b1a78aSEric Anholt struct drm_file *file_priv) 1091d5b1a78aSEric Anholt { 1092d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 109365101d8cSBoris Brezillon struct vc4_file *vc4file = file_priv->driver_priv; 1094d5b1a78aSEric Anholt struct drm_vc4_submit_cl *args = data; 1095d5b1a78aSEric Anholt struct vc4_exec_info *exec; 1096cdec4d36SEric Anholt struct ww_acquire_ctx acquire_ctx; 109736cb6253SEric Anholt int ret = 0; 1098d5b1a78aSEric Anholt 10993be8edddSEric Anholt if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR | 11003be8edddSEric Anholt VC4_SUBMIT_CL_FIXED_RCL_ORDER | 11013be8edddSEric Anholt VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X | 11023be8edddSEric Anholt VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y)) != 0) { 1103fb95992aSEric Anholt DRM_DEBUG("Unknown flags: 0x%02x\n", args->flags); 1104d5b1a78aSEric Anholt return -EINVAL; 1105d5b1a78aSEric Anholt } 1106d5b1a78aSEric Anholt 110765101d8cSBoris Brezillon if (args->pad2 != 0) { 110865101d8cSBoris Brezillon DRM_DEBUG("->pad2 must be set to zero\n"); 110965101d8cSBoris Brezillon return -EINVAL; 111065101d8cSBoris Brezillon } 111165101d8cSBoris Brezillon 1112d5b1a78aSEric Anholt exec = kcalloc(1, sizeof(*exec), GFP_KERNEL); 1113d5b1a78aSEric Anholt if (!exec) { 1114d5b1a78aSEric Anholt DRM_ERROR("malloc failure on exec struct\n"); 1115d5b1a78aSEric Anholt return -ENOMEM; 1116d5b1a78aSEric Anholt } 1117d5b1a78aSEric Anholt 111836cb6253SEric Anholt mutex_lock(&vc4->power_lock); 1119925d05e1SEric Anholt if (vc4->power_refcount++ == 0) { 1120001bdb55SEric Anholt ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev); 1121001bdb55SEric Anholt if (ret < 0) { 1122925d05e1SEric Anholt mutex_unlock(&vc4->power_lock); 1123925d05e1SEric Anholt vc4->power_refcount--; 1124001bdb55SEric Anholt kfree(exec); 1125001bdb55SEric Anholt return ret; 1126001bdb55SEric Anholt } 1127925d05e1SEric Anholt } 1128925d05e1SEric Anholt mutex_unlock(&vc4->power_lock); 1129001bdb55SEric Anholt 1130d5b1a78aSEric Anholt exec->args = args; 1131d5b1a78aSEric Anholt INIT_LIST_HEAD(&exec->unref_list); 1132d5b1a78aSEric Anholt 1133d5b1a78aSEric Anholt ret = vc4_cl_lookup_bos(dev, file_priv, exec); 1134d5b1a78aSEric Anholt if (ret) 1135d5b1a78aSEric Anholt goto fail; 1136d5b1a78aSEric Anholt 113765101d8cSBoris Brezillon if (args->perfmonid) { 113865101d8cSBoris Brezillon exec->perfmon = vc4_perfmon_find(vc4file, 113965101d8cSBoris Brezillon args->perfmonid); 114065101d8cSBoris Brezillon if (!exec->perfmon) { 114165101d8cSBoris Brezillon ret = -ENOENT; 114265101d8cSBoris Brezillon goto fail; 114365101d8cSBoris Brezillon } 114465101d8cSBoris Brezillon } 114565101d8cSBoris Brezillon 1146d5b1a78aSEric Anholt if (exec->args->bin_cl_size != 0) { 1147d5b1a78aSEric Anholt ret = vc4_get_bcl(dev, exec); 1148d5b1a78aSEric Anholt if (ret) 1149d5b1a78aSEric Anholt goto fail; 1150d5b1a78aSEric Anholt } else { 1151d5b1a78aSEric Anholt exec->ct0ca = 0; 1152d5b1a78aSEric Anholt exec->ct0ea = 0; 1153d5b1a78aSEric Anholt } 1154d5b1a78aSEric Anholt 1155d5b1a78aSEric Anholt ret = vc4_get_rcl(dev, exec); 1156d5b1a78aSEric Anholt if (ret) 1157d5b1a78aSEric Anholt goto fail; 1158d5b1a78aSEric Anholt 1159cdec4d36SEric Anholt ret = vc4_lock_bo_reservations(dev, exec, &acquire_ctx); 1160cdec4d36SEric Anholt if (ret) 1161cdec4d36SEric Anholt goto fail; 1162cdec4d36SEric Anholt 1163d5b1a78aSEric Anholt /* Clear this out of the struct we'll be putting in the queue, 1164d5b1a78aSEric Anholt * since it's part of our stack. 1165d5b1a78aSEric Anholt */ 1166d5b1a78aSEric Anholt exec->args = NULL; 1167d5b1a78aSEric Anholt 1168cdec4d36SEric Anholt ret = vc4_queue_submit(dev, exec, &acquire_ctx); 1169cdec4d36SEric Anholt if (ret) 1170cdec4d36SEric Anholt goto fail; 1171d5b1a78aSEric Anholt 1172d5b1a78aSEric Anholt /* Return the seqno for our job. */ 1173d5b1a78aSEric Anholt args->seqno = vc4->emit_seqno; 1174d5b1a78aSEric Anholt 1175d5b1a78aSEric Anholt return 0; 1176d5b1a78aSEric Anholt 1177d5b1a78aSEric Anholt fail: 1178d5b1a78aSEric Anholt vc4_complete_exec(vc4->dev, exec); 1179d5b1a78aSEric Anholt 1180d5b1a78aSEric Anholt return ret; 1181d5b1a78aSEric Anholt } 1182d5b1a78aSEric Anholt 1183d5b1a78aSEric Anholt void 1184d5b1a78aSEric Anholt vc4_gem_init(struct drm_device *dev) 1185d5b1a78aSEric Anholt { 1186d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 1187d5b1a78aSEric Anholt 1188cdec4d36SEric Anholt vc4->dma_fence_context = dma_fence_context_alloc(1); 1189cdec4d36SEric Anholt 1190ca26d28bSVarad Gautam INIT_LIST_HEAD(&vc4->bin_job_list); 1191ca26d28bSVarad Gautam INIT_LIST_HEAD(&vc4->render_job_list); 1192d5b1a78aSEric Anholt INIT_LIST_HEAD(&vc4->job_done_list); 1193b501baccSEric Anholt INIT_LIST_HEAD(&vc4->seqno_cb_list); 1194d5b1a78aSEric Anholt spin_lock_init(&vc4->job_lock); 1195d5b1a78aSEric Anholt 1196d5b1a78aSEric Anholt INIT_WORK(&vc4->hangcheck.reset_work, vc4_reset_work); 11970078730fSKees Cook timer_setup(&vc4->hangcheck.timer, vc4_hangcheck_elapsed, 0); 1198d5b1a78aSEric Anholt 1199d5b1a78aSEric Anholt INIT_WORK(&vc4->job_done_work, vc4_job_done_work); 120036cb6253SEric Anholt 120136cb6253SEric Anholt mutex_init(&vc4->power_lock); 1202b9f19259SBoris Brezillon 1203b9f19259SBoris Brezillon INIT_LIST_HEAD(&vc4->purgeable.list); 1204b9f19259SBoris Brezillon mutex_init(&vc4->purgeable.lock); 1205d5b1a78aSEric Anholt } 1206d5b1a78aSEric Anholt 1207d5b1a78aSEric Anholt void 1208d5b1a78aSEric Anholt vc4_gem_destroy(struct drm_device *dev) 1209d5b1a78aSEric Anholt { 1210d5b1a78aSEric Anholt struct vc4_dev *vc4 = to_vc4_dev(dev); 1211d5b1a78aSEric Anholt 1212d5b1a78aSEric Anholt /* Waiting for exec to finish would need to be done before 1213d5b1a78aSEric Anholt * unregistering V3D. 1214d5b1a78aSEric Anholt */ 1215d5b1a78aSEric Anholt WARN_ON(vc4->emit_seqno != vc4->finished_seqno); 1216d5b1a78aSEric Anholt 1217d5b1a78aSEric Anholt /* V3D should already have disabled its interrupt and cleared 1218d5b1a78aSEric Anholt * the overflow allocation registers. Now free the object. 1219d5b1a78aSEric Anholt */ 1220553c942fSEric Anholt if (vc4->bin_bo) { 1221553c942fSEric Anholt drm_gem_object_put_unlocked(&vc4->bin_bo->base.base); 1222553c942fSEric Anholt vc4->bin_bo = NULL; 1223d5b1a78aSEric Anholt } 1224d5b1a78aSEric Anholt 122521461365SEric Anholt if (vc4->hang_state) 122621461365SEric Anholt vc4_free_hang_state(dev, vc4->hang_state); 1227d5b1a78aSEric Anholt } 1228b9f19259SBoris Brezillon 1229b9f19259SBoris Brezillon int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data, 1230b9f19259SBoris Brezillon struct drm_file *file_priv) 1231b9f19259SBoris Brezillon { 1232b9f19259SBoris Brezillon struct drm_vc4_gem_madvise *args = data; 1233b9f19259SBoris Brezillon struct drm_gem_object *gem_obj; 1234b9f19259SBoris Brezillon struct vc4_bo *bo; 1235b9f19259SBoris Brezillon int ret; 1236b9f19259SBoris Brezillon 1237b9f19259SBoris Brezillon switch (args->madv) { 1238b9f19259SBoris Brezillon case VC4_MADV_DONTNEED: 1239b9f19259SBoris Brezillon case VC4_MADV_WILLNEED: 1240b9f19259SBoris Brezillon break; 1241b9f19259SBoris Brezillon default: 1242b9f19259SBoris Brezillon return -EINVAL; 1243b9f19259SBoris Brezillon } 1244b9f19259SBoris Brezillon 1245b9f19259SBoris Brezillon if (args->pad != 0) 1246b9f19259SBoris Brezillon return -EINVAL; 1247b9f19259SBoris Brezillon 1248b9f19259SBoris Brezillon gem_obj = drm_gem_object_lookup(file_priv, args->handle); 1249b9f19259SBoris Brezillon if (!gem_obj) { 1250b9f19259SBoris Brezillon DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 1251b9f19259SBoris Brezillon return -ENOENT; 1252b9f19259SBoris Brezillon } 1253b9f19259SBoris Brezillon 1254b9f19259SBoris Brezillon bo = to_vc4_bo(gem_obj); 1255b9f19259SBoris Brezillon 1256b9f19259SBoris Brezillon /* Only BOs exposed to userspace can be purged. */ 1257b9f19259SBoris Brezillon if (bo->madv == __VC4_MADV_NOTSUPP) { 1258b9f19259SBoris Brezillon DRM_DEBUG("madvise not supported on this BO\n"); 1259b9f19259SBoris Brezillon ret = -EINVAL; 1260b9f19259SBoris Brezillon goto out_put_gem; 1261b9f19259SBoris Brezillon } 1262b9f19259SBoris Brezillon 1263b9f19259SBoris Brezillon /* Not sure it's safe to purge imported BOs. Let's just assume it's 1264b9f19259SBoris Brezillon * not until proven otherwise. 1265b9f19259SBoris Brezillon */ 1266b9f19259SBoris Brezillon if (gem_obj->import_attach) { 1267b9f19259SBoris Brezillon DRM_DEBUG("madvise not supported on imported BOs\n"); 1268b9f19259SBoris Brezillon ret = -EINVAL; 1269b9f19259SBoris Brezillon goto out_put_gem; 1270b9f19259SBoris Brezillon } 1271b9f19259SBoris Brezillon 1272b9f19259SBoris Brezillon mutex_lock(&bo->madv_lock); 1273b9f19259SBoris Brezillon 1274b9f19259SBoris Brezillon if (args->madv == VC4_MADV_DONTNEED && bo->madv == VC4_MADV_WILLNEED && 1275b9f19259SBoris Brezillon !refcount_read(&bo->usecnt)) { 1276b9f19259SBoris Brezillon /* If the BO is about to be marked as purgeable, is not used 1277b9f19259SBoris Brezillon * and is not already purgeable or purged, add it to the 1278b9f19259SBoris Brezillon * purgeable list. 1279b9f19259SBoris Brezillon */ 1280b9f19259SBoris Brezillon vc4_bo_add_to_purgeable_pool(bo); 1281b9f19259SBoris Brezillon } else if (args->madv == VC4_MADV_WILLNEED && 1282b9f19259SBoris Brezillon bo->madv == VC4_MADV_DONTNEED && 1283b9f19259SBoris Brezillon !refcount_read(&bo->usecnt)) { 1284b9f19259SBoris Brezillon /* The BO has not been purged yet, just remove it from 1285b9f19259SBoris Brezillon * the purgeable list. 1286b9f19259SBoris Brezillon */ 1287b9f19259SBoris Brezillon vc4_bo_remove_from_purgeable_pool(bo); 1288b9f19259SBoris Brezillon } 1289b9f19259SBoris Brezillon 1290b9f19259SBoris Brezillon /* Save the purged state. */ 1291b9f19259SBoris Brezillon args->retained = bo->madv != __VC4_MADV_PURGED; 1292b9f19259SBoris Brezillon 1293b9f19259SBoris Brezillon /* Update internal madv state only if the bo was not purged. */ 1294b9f19259SBoris Brezillon if (bo->madv != __VC4_MADV_PURGED) 1295b9f19259SBoris Brezillon bo->madv = args->madv; 1296b9f19259SBoris Brezillon 1297b9f19259SBoris Brezillon mutex_unlock(&bo->madv_lock); 1298b9f19259SBoris Brezillon 1299b9f19259SBoris Brezillon ret = 0; 1300b9f19259SBoris Brezillon 1301b9f19259SBoris Brezillon out_put_gem: 1302b9f19259SBoris Brezillon drm_gem_object_put_unlocked(gem_obj); 1303b9f19259SBoris Brezillon 1304b9f19259SBoris Brezillon return ret; 1305b9f19259SBoris Brezillon } 1306