180 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)  in ivpu_cmdq_push_job()  argument
182 	struct ivpu_device *vdev = job->vdev;  in ivpu_cmdq_push_job()
191 			 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);  in ivpu_cmdq_push_job()
195 	entry = &cmdq->jobq->job[tail];  in ivpu_cmdq_push_job()
196 	entry->batch_buf_addr = job->cmd_buf_vpu_addr;  in ivpu_cmdq_push_job()
197 	entry->job_id = job->job_id;  in ivpu_cmdq_push_job()
249 static void job_get(struct ivpu_job *job, struct ivpu_job **link)  in job_get()  argument
251 	struct ivpu_device *vdev = job->vdev;  in job_get()
253 	kref_get(&job->ref);  in job_get()
254 	*link = job;  in job_get()
256 	ivpu_dbg(vdev, KREF, "Job get: id %u refcount %u\n", job->job_id, kref_read(&job->ref));  in job_get()
261 	struct ivpu_job *job = container_of(ref, struct ivpu_job, ref);  in job_release()  local
262 	struct ivpu_device *vdev = job->vdev;  in job_release()
265 	for (i = 0; i < job->bo_count; i++)  in job_release()
266 		if (job->bos[i])  in job_release()
267 			drm_gem_object_put(&job->bos[i]->base);  in job_release()
269 	dma_fence_put(job->done_fence);  in job_release()
270 	ivpu_file_priv_put(&job->file_priv);  in job_release()
272 	ivpu_dbg(vdev, KREF, "Job released: id %u\n", job->job_id);  in job_release()
273 	kfree(job);  in job_release()
279 static void job_put(struct ivpu_job *job)  in job_put()  argument
281 	struct ivpu_device *vdev = job->vdev;  in job_put()
283 	ivpu_dbg(vdev, KREF, "Job put: id %u refcount %u\n", job->job_id, kref_read(&job->ref));  in job_put()
284 	kref_put(&job->ref, job_release);  in job_put()
291 	struct ivpu_job *job;  in ivpu_create_job()  local
298 	job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);  in ivpu_create_job()
299 	if (!job)  in ivpu_create_job()
302 	kref_init(&job->ref);  in ivpu_create_job()
304 	job->vdev = vdev;  in ivpu_create_job()
305 	job->engine_idx = engine_idx;  in ivpu_create_job()
306 	job->bo_count = bo_count;  in ivpu_create_job()
307 	job->done_fence = ivpu_fence_create(vdev);  in ivpu_create_job()
308 	if (!job->done_fence) {  in ivpu_create_job()
313 	job->file_priv = ivpu_file_priv_get(file_priv);  in ivpu_create_job()
315 	ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);  in ivpu_create_job()
317 	return job;  in ivpu_create_job()
320 	kfree(job);  in ivpu_create_job()
328 	struct ivpu_job *job;  in ivpu_job_done()  local
330 	job = xa_erase(&vdev->submitted_jobs_xa, job_id);  in ivpu_job_done()
331 	if (!job)  in ivpu_job_done()
334 	if (job->file_priv->has_mmu_faults)  in ivpu_job_done()
337 	job->bos[CMD_BUF_IDX]->job_status = job_status;  in ivpu_job_done()
338 	dma_fence_signal(job->done_fence);  in ivpu_job_done()
341 		 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);  in ivpu_job_done()
343 	job_put(job);  in ivpu_job_done()
362 	struct ivpu_job *job;  in ivpu_jobs_abort_all()  local
365 	xa_for_each(&vdev->submitted_jobs_xa, id, job)  in ivpu_jobs_abort_all()
369 static int ivpu_direct_job_submission(struct ivpu_job *job)  in ivpu_direct_job_submission()  argument
371 	struct ivpu_file_priv *file_priv = job->file_priv;  in ivpu_direct_job_submission()
372 	struct ivpu_device *vdev = job->vdev;  in ivpu_direct_job_submission()
379 	cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);  in ivpu_direct_job_submission()
382 			  file_priv->ctx.id, job->engine_idx);  in ivpu_direct_job_submission()
390 	job_get(job, &job);  in ivpu_direct_job_submission()
391 	ret = xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);  in ivpu_direct_job_submission()
397 	ret = ivpu_cmdq_push_job(cmdq, job);  in ivpu_direct_job_submission()
402 		 job->job_id, job->cmd_buf_vpu_addr, file_priv->ctx.id,  in ivpu_direct_job_submission()
403 		 job->engine_idx, cmdq->jobq->header.tail);  in ivpu_direct_job_submission()
406 		ivpu_job_done(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);  in ivpu_direct_job_submission()
417 	xa_erase(&vdev->submitted_jobs_xa, job->job_id);  in ivpu_direct_job_submission()
419 	job_put(job);  in ivpu_direct_job_submission()
426 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,  in ivpu_job_prepare_bos_for_submit()  argument
443 		job->bos[i] = to_ivpu_bo(obj);  in ivpu_job_prepare_bos_for_submit()
445 		ret = ivpu_bo_pin(job->bos[i]);  in ivpu_job_prepare_bos_for_submit()
450 	bo = job->bos[CMD_BUF_IDX];  in ivpu_job_prepare_bos_for_submit()
461 	job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;  in ivpu_job_prepare_bos_for_submit()
463 	ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,  in ivpu_job_prepare_bos_for_submit()
471 		ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1);  in ivpu_job_prepare_bos_for_submit()
480 		dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, usage);  in ivpu_job_prepare_bos_for_submit()
484 	drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);  in ivpu_job_prepare_bos_for_submit()
496 	struct ivpu_job *job;  in ivpu_submit_ioctl()  local
535 	job = ivpu_create_job(file_priv, params->engine, params->buffer_count);  in ivpu_submit_ioctl()
536 	if (!job) {  in ivpu_submit_ioctl()
542 	ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,  in ivpu_submit_ioctl()
549 	ret = ivpu_direct_job_submission(job);  in ivpu_submit_ioctl()
551 		dma_fence_signal(job->done_fence);  in ivpu_submit_ioctl()
556 	job_put(job);  in ivpu_submit_ioctl()