106 int panfrost_job_get_slot(struct panfrost_job *job)  in panfrost_job_get_slot()  argument
112 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
117 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
118 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
119 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
121 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
146 panfrost_get_job_chain_flag(const struct panfrost_job *job)  in panfrost_get_job_chain_flag()  argument
148 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);  in panfrost_get_job_chain_flag()
150 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
159 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()  local
161 	WARN_ON(!job);  in panfrost_dequeue_job()
165 	return job;  in panfrost_dequeue_job()
170 		     struct panfrost_job *job)  in panfrost_enqueue_job()  argument
172 	if (WARN_ON(!job))  in panfrost_enqueue_job()
176 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
181 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
182 	WARN_ON(panfrost_get_job_chain_flag(job) ==  in panfrost_enqueue_job()
187 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)  in panfrost_job_hw_submit()  argument
189 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
192 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
205 	cfg = panfrost_mmu_as_get(pfdev, job->mmu);  in panfrost_job_hw_submit()
210 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
217 		panfrost_get_job_chain_flag(job);  in panfrost_job_hw_submit()
228 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
233 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
239 			job, js, subslot, jc_head, cfg & 0xf);  in panfrost_job_hw_submit()
246 					  struct drm_sched_job *job)  in panfrost_acquire_object_fences()  argument
256 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],  in panfrost_acquire_object_fences()
275 int panfrost_job_push(struct panfrost_job *job)  in panfrost_job_push()  argument
277 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
281 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
287 	drm_sched_job_arm(&job->base);  in panfrost_job_push()
289 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
291 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
292 					     &job->base);  in panfrost_job_push()
298 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
300 	drm_sched_entity_push_job(&job->base);  in panfrost_job_push()
304 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
305 				      job->render_done_fence);  in panfrost_job_push()
308 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
315 	struct panfrost_job *job = container_of(ref, struct panfrost_job,  in panfrost_job_cleanup()  local
319 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
320 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
322 	if (job->mappings) {  in panfrost_job_cleanup()
323 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
324 			if (!job->mappings[i])  in panfrost_job_cleanup()
327 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
328 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
330 		kvfree(job->mappings);  in panfrost_job_cleanup()
333 	if (job->bos) {  in panfrost_job_cleanup()
334 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
335 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
337 		kvfree(job->bos);  in panfrost_job_cleanup()
340 	kfree(job);  in panfrost_job_cleanup()
343 void panfrost_job_put(struct panfrost_job *job)  in panfrost_job_put()  argument
345 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
350 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_free()  local
354 	panfrost_job_put(job);  in panfrost_job_free()
359 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_run()  local
360 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
361 	int slot = panfrost_job_get_slot(job);  in panfrost_job_run()
364 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
370 	if (!job->jc)  in panfrost_job_run()
377 	if (job->done_fence)  in panfrost_job_run()
378 		dma_fence_put(job->done_fence);  in panfrost_job_run()
379 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
381 	panfrost_job_hw_submit(job, slot);  in panfrost_job_run()
400 				    struct panfrost_job *job,  in panfrost_job_handle_err()  argument
421 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
428 		dma_fence_set_error(job->done_fence, -ECANCELED);  in panfrost_job_handle_err()
429 		job->jc = 0;  in panfrost_job_handle_err()
435 		dma_fence_set_error(job->done_fence, -EINVAL);  in panfrost_job_handle_err()
436 		job->jc = 0;  in panfrost_job_handle_err()
439 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_err()
443 		dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_err()
454 				     struct panfrost_job *job)  in panfrost_job_handle_done()  argument
459 	job->jc = 0;  in panfrost_job_handle_done()
460 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_done()
463 	dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_done()
712 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_timedout()  local
713 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
714 	int js = panfrost_job_get_slot(job);  in panfrost_job_timedout()
720 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
734 	if (dma_fence_is_signaled(job->done_fence)) {  in panfrost_job_timedout()
747 	panfrost_core_dump(job);  in panfrost_job_timedout()
905 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()  local
908 			if (!job || job->base.entity != entity)  in panfrost_job_close()
917 				job->jc = 0;  in panfrost_job_close()
921 				cmd = panfrost_get_job_chain_flag(job) ?  in panfrost_job_close()