xref: /openbmc/linux/drivers/accel/ivpu/ivpu_gem.c (revision 162f17b2)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11 
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16 
17 #include "ivpu_drv.h"
18 #include "ivpu_gem.h"
19 #include "ivpu_hw.h"
20 #include "ivpu_mmu.h"
21 #include "ivpu_mmu_context.h"
22 
23 MODULE_IMPORT_NS(DMA_BUF);
24 
25 static const struct drm_gem_object_funcs ivpu_gem_funcs;
26 
27 static struct lock_class_key prime_bo_lock_class_key;
28 
prime_alloc_pages_locked(struct ivpu_bo * bo)29 static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo)
30 {
31 	/* Pages are managed by the underlying dma-buf */
32 	return 0;
33 }
34 
prime_free_pages_locked(struct ivpu_bo * bo)35 static void prime_free_pages_locked(struct ivpu_bo *bo)
36 {
37 	/* Pages are managed by the underlying dma-buf */
38 }
39 
prime_map_pages_locked(struct ivpu_bo * bo)40 static int prime_map_pages_locked(struct ivpu_bo *bo)
41 {
42 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
43 	struct sg_table *sgt;
44 
45 	sgt = dma_buf_map_attachment_unlocked(bo->base.import_attach, DMA_BIDIRECTIONAL);
46 	if (IS_ERR(sgt)) {
47 		ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt));
48 		return PTR_ERR(sgt);
49 	}
50 
51 	bo->sgt = sgt;
52 	return 0;
53 }
54 
prime_unmap_pages_locked(struct ivpu_bo * bo)55 static void prime_unmap_pages_locked(struct ivpu_bo *bo)
56 {
57 	dma_buf_unmap_attachment_unlocked(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL);
58 	bo->sgt = NULL;
59 }
60 
61 static const struct ivpu_bo_ops prime_ops = {
62 	.type = IVPU_BO_TYPE_PRIME,
63 	.name = "prime",
64 	.alloc_pages = prime_alloc_pages_locked,
65 	.free_pages = prime_free_pages_locked,
66 	.map_pages = prime_map_pages_locked,
67 	.unmap_pages = prime_unmap_pages_locked,
68 };
69 
shmem_alloc_pages_locked(struct ivpu_bo * bo)70 static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo)
71 {
72 	int npages = bo->base.size >> PAGE_SHIFT;
73 	struct page **pages;
74 
75 	pages = drm_gem_get_pages(&bo->base);
76 	if (IS_ERR(pages))
77 		return PTR_ERR(pages);
78 
79 	if (bo->flags & DRM_IVPU_BO_WC)
80 		set_pages_array_wc(pages, npages);
81 	else if (bo->flags & DRM_IVPU_BO_UNCACHED)
82 		set_pages_array_uc(pages, npages);
83 
84 	bo->pages = pages;
85 	return 0;
86 }
87 
shmem_free_pages_locked(struct ivpu_bo * bo)88 static void shmem_free_pages_locked(struct ivpu_bo *bo)
89 {
90 	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
91 		set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT);
92 
93 	drm_gem_put_pages(&bo->base, bo->pages, true, false);
94 	bo->pages = NULL;
95 }
96 
ivpu_bo_map_pages_locked(struct ivpu_bo * bo)97 static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo)
98 {
99 	int npages = bo->base.size >> PAGE_SHIFT;
100 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
101 	struct sg_table *sgt;
102 	int ret;
103 
104 	sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages);
105 	if (IS_ERR(sgt)) {
106 		ivpu_err(vdev, "Failed to allocate sgtable\n");
107 		return PTR_ERR(sgt);
108 	}
109 
110 	ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0);
111 	if (ret) {
112 		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
113 		goto err_free_sgt;
114 	}
115 
116 	bo->sgt = sgt;
117 	return 0;
118 
119 err_free_sgt:
120 	kfree(sgt);
121 	return ret;
122 }
123 
ivpu_bo_unmap_pages_locked(struct ivpu_bo * bo)124 static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo)
125 {
126 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
127 
128 	dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0);
129 	sg_free_table(bo->sgt);
130 	kfree(bo->sgt);
131 	bo->sgt = NULL;
132 }
133 
134 static const struct ivpu_bo_ops shmem_ops = {
135 	.type = IVPU_BO_TYPE_SHMEM,
136 	.name = "shmem",
137 	.alloc_pages = shmem_alloc_pages_locked,
138 	.free_pages = shmem_free_pages_locked,
139 	.map_pages = ivpu_bo_map_pages_locked,
140 	.unmap_pages = ivpu_bo_unmap_pages_locked,
141 };
142 
internal_alloc_pages_locked(struct ivpu_bo * bo)143 static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo)
144 {
145 	unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
146 	struct page **pages;
147 	int ret;
148 
149 	pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL);
150 	if (!pages)
151 		return -ENOMEM;
152 
153 	for (i = 0; i < npages; i++) {
154 		pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
155 		if (!pages[i]) {
156 			ret = -ENOMEM;
157 			goto err_free_pages;
158 		}
159 		cond_resched();
160 	}
161 
162 	bo->pages = pages;
163 	return 0;
164 
165 err_free_pages:
166 	while (i--)
167 		put_page(pages[i]);
168 	kvfree(pages);
169 	return ret;
170 }
171 
internal_free_pages_locked(struct ivpu_bo * bo)172 static void internal_free_pages_locked(struct ivpu_bo *bo)
173 {
174 	unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
175 
176 	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
177 		set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT);
178 
179 	for (i = 0; i < npages; i++)
180 		put_page(bo->pages[i]);
181 
182 	kvfree(bo->pages);
183 	bo->pages = NULL;
184 }
185 
186 static const struct ivpu_bo_ops internal_ops = {
187 	.type = IVPU_BO_TYPE_INTERNAL,
188 	.name = "internal",
189 	.alloc_pages = internal_alloc_pages_locked,
190 	.free_pages = internal_free_pages_locked,
191 	.map_pages = ivpu_bo_map_pages_locked,
192 	.unmap_pages = ivpu_bo_unmap_pages_locked,
193 };
194 
ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo * bo)195 static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo)
196 {
197 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
198 	int ret;
199 
200 	lockdep_assert_held(&bo->lock);
201 	drm_WARN_ON(&vdev->drm, bo->sgt);
202 
203 	ret = bo->ops->alloc_pages(bo);
204 	if (ret) {
205 		ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret);
206 		return ret;
207 	}
208 
209 	ret = bo->ops->map_pages(bo);
210 	if (ret) {
211 		ivpu_err(vdev, "Failed to map pages for BO: %d", ret);
212 		goto err_free_pages;
213 	}
214 	return ret;
215 
216 err_free_pages:
217 	bo->ops->free_pages(bo);
218 	return ret;
219 }
220 
ivpu_bo_unmap_and_free_pages(struct ivpu_bo * bo)221 static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo)
222 {
223 	mutex_lock(&bo->lock);
224 
225 	WARN_ON(!bo->sgt);
226 	bo->ops->unmap_pages(bo);
227 	WARN_ON(bo->sgt);
228 	bo->ops->free_pages(bo);
229 	WARN_ON(bo->pages);
230 
231 	mutex_unlock(&bo->lock);
232 }
233 
234 /*
235  * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
236  *
237  * This function pins physical memory pages, then maps the physical pages
238  * to IOMMU address space and finally updates the VPU MMU page tables
239  * to allow the VPU to translate VPU address to IOMMU address.
240  */
ivpu_bo_pin(struct ivpu_bo * bo)241 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
242 {
243 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
244 	int ret = 0;
245 
246 	mutex_lock(&bo->lock);
247 
248 	if (!bo->vpu_addr) {
249 		ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n",
250 			 bo->ctx->id, bo->handle);
251 		ret = -EINVAL;
252 		goto unlock;
253 	}
254 
255 	if (!bo->sgt) {
256 		ret = ivpu_bo_alloc_and_map_pages_locked(bo);
257 		if (ret)
258 			goto unlock;
259 	}
260 
261 	if (!bo->mmu_mapped) {
262 		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt,
263 					       ivpu_bo_is_snooped(bo));
264 		if (ret) {
265 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
266 			goto unlock;
267 		}
268 		bo->mmu_mapped = true;
269 	}
270 
271 unlock:
272 	mutex_unlock(&bo->lock);
273 
274 	return ret;
275 }
276 
277 static int
ivpu_bo_alloc_vpu_addr(struct ivpu_bo * bo,struct ivpu_mmu_context * ctx,const struct ivpu_addr_range * range)278 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
279 		       const struct ivpu_addr_range *range)
280 {
281 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
282 	int ret;
283 
284 	if (!range) {
285 		if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
286 			range = &vdev->hw->ranges.shave;
287 		else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
288 			range = &vdev->hw->ranges.dma;
289 		else
290 			range = &vdev->hw->ranges.user;
291 	}
292 
293 	mutex_lock(&ctx->lock);
294 	ret = ivpu_mmu_context_insert_node_locked(ctx, range, bo->base.size, &bo->mm_node);
295 	if (!ret) {
296 		bo->ctx = ctx;
297 		bo->vpu_addr = bo->mm_node.start;
298 		list_add_tail(&bo->ctx_node, &ctx->bo_list);
299 	}
300 	mutex_unlock(&ctx->lock);
301 
302 	return ret;
303 }
304 
ivpu_bo_free_vpu_addr(struct ivpu_bo * bo)305 static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo)
306 {
307 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
308 	struct ivpu_mmu_context *ctx = bo->ctx;
309 
310 	ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
311 		 ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
312 
313 	mutex_lock(&bo->lock);
314 
315 	if (bo->mmu_mapped) {
316 		drm_WARN_ON(&vdev->drm, !bo->sgt);
317 		ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt);
318 		bo->mmu_mapped = false;
319 	}
320 
321 	mutex_lock(&ctx->lock);
322 	list_del(&bo->ctx_node);
323 	bo->vpu_addr = 0;
324 	bo->ctx = NULL;
325 	ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node);
326 	mutex_unlock(&ctx->lock);
327 
328 	mutex_unlock(&bo->lock);
329 }
330 
ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context * ctx)331 void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx)
332 {
333 	struct ivpu_bo *bo, *tmp;
334 
335 	list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node)
336 		ivpu_bo_free_vpu_addr(bo);
337 }
338 
339 static struct ivpu_bo *
ivpu_bo_alloc(struct ivpu_device * vdev,struct ivpu_mmu_context * mmu_context,u64 size,u32 flags,const struct ivpu_bo_ops * ops,const struct ivpu_addr_range * range,u64 user_ptr)340 ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context,
341 	      u64 size, u32 flags, const struct ivpu_bo_ops *ops,
342 	      const struct ivpu_addr_range *range, u64 user_ptr)
343 {
344 	struct ivpu_bo *bo;
345 	int ret = 0;
346 
347 	if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size)))
348 		return ERR_PTR(-EINVAL);
349 
350 	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
351 	case DRM_IVPU_BO_CACHED:
352 	case DRM_IVPU_BO_UNCACHED:
353 	case DRM_IVPU_BO_WC:
354 		break;
355 	default:
356 		return ERR_PTR(-EINVAL);
357 	}
358 
359 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
360 	if (!bo)
361 		return ERR_PTR(-ENOMEM);
362 
363 	mutex_init(&bo->lock);
364 	bo->base.funcs = &ivpu_gem_funcs;
365 	bo->flags = flags;
366 	bo->ops = ops;
367 	bo->user_ptr = user_ptr;
368 
369 	if (ops->type == IVPU_BO_TYPE_SHMEM)
370 		ret = drm_gem_object_init(&vdev->drm, &bo->base, size);
371 	else
372 		drm_gem_private_object_init(&vdev->drm, &bo->base, size);
373 
374 	if (ret) {
375 		ivpu_err(vdev, "Failed to initialize drm object\n");
376 		goto err_free;
377 	}
378 
379 	if (flags & DRM_IVPU_BO_MAPPABLE) {
380 		ret = drm_gem_create_mmap_offset(&bo->base);
381 		if (ret) {
382 			ivpu_err(vdev, "Failed to allocate mmap offset\n");
383 			goto err_release;
384 		}
385 	}
386 
387 	if (mmu_context) {
388 		ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range);
389 		if (ret) {
390 			ivpu_err(vdev, "Failed to add BO to context: %d\n", ret);
391 			goto err_release;
392 		}
393 	}
394 
395 	return bo;
396 
397 err_release:
398 	drm_gem_object_release(&bo->base);
399 err_free:
400 	kfree(bo);
401 	return ERR_PTR(ret);
402 }
403 
ivpu_bo_free(struct drm_gem_object * obj)404 static void ivpu_bo_free(struct drm_gem_object *obj)
405 {
406 	struct ivpu_bo *bo = to_ivpu_bo(obj);
407 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
408 
409 	if (bo->ctx)
410 		ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
411 			 bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
412 	else
413 		ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n",
414 			 (bool)bo->sgt, bo->mmu_mapped);
415 
416 	drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
417 
418 	vunmap(bo->kvaddr);
419 
420 	if (bo->ctx)
421 		ivpu_bo_free_vpu_addr(bo);
422 
423 	if (bo->sgt)
424 		ivpu_bo_unmap_and_free_pages(bo);
425 
426 	if (bo->base.import_attach)
427 		drm_prime_gem_destroy(&bo->base, bo->sgt);
428 
429 	drm_gem_object_release(&bo->base);
430 
431 	mutex_destroy(&bo->lock);
432 	kfree(bo);
433 }
434 
ivpu_bo_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)435 static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
436 {
437 	struct ivpu_bo *bo = to_ivpu_bo(obj);
438 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
439 
440 	ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s",
441 		 bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size, bo->ops->name);
442 
443 	if (obj->import_attach) {
444 		/* Drop the reference drm_gem_mmap_obj() acquired.*/
445 		drm_gem_object_put(obj);
446 		vma->vm_private_data = NULL;
447 		return dma_buf_mmap(obj->dma_buf, vma, 0);
448 	}
449 
450 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND);
451 	vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags));
452 
453 	return 0;
454 }
455 
ivpu_bo_get_sg_table(struct drm_gem_object * obj)456 static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj)
457 {
458 	struct ivpu_bo *bo = to_ivpu_bo(obj);
459 	loff_t npages = obj->size >> PAGE_SHIFT;
460 	int ret = 0;
461 
462 	mutex_lock(&bo->lock);
463 
464 	if (!bo->sgt)
465 		ret = ivpu_bo_alloc_and_map_pages_locked(bo);
466 
467 	mutex_unlock(&bo->lock);
468 
469 	if (ret)
470 		return ERR_PTR(ret);
471 
472 	return drm_prime_pages_to_sg(obj->dev, bo->pages, npages);
473 }
474 
ivpu_vm_fault(struct vm_fault * vmf)475 static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf)
476 {
477 	struct vm_area_struct *vma = vmf->vma;
478 	struct drm_gem_object *obj = vma->vm_private_data;
479 	struct ivpu_bo *bo = to_ivpu_bo(obj);
480 	loff_t npages = obj->size >> PAGE_SHIFT;
481 	pgoff_t page_offset;
482 	struct page *page;
483 	vm_fault_t ret;
484 	int err;
485 
486 	mutex_lock(&bo->lock);
487 
488 	if (!bo->sgt) {
489 		err = ivpu_bo_alloc_and_map_pages_locked(bo);
490 		if (err) {
491 			ret = vmf_error(err);
492 			goto unlock;
493 		}
494 	}
495 
496 	/* We don't use vmf->pgoff since that has the fake offset */
497 	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
498 	if (page_offset >= npages) {
499 		ret = VM_FAULT_SIGBUS;
500 	} else {
501 		page = bo->pages[page_offset];
502 		ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
503 	}
504 
505 unlock:
506 	mutex_unlock(&bo->lock);
507 
508 	return ret;
509 }
510 
511 static const struct vm_operations_struct ivpu_vm_ops = {
512 	.fault = ivpu_vm_fault,
513 	.open = drm_gem_vm_open,
514 	.close = drm_gem_vm_close,
515 };
516 
517 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
518 	.free = ivpu_bo_free,
519 	.mmap = ivpu_bo_mmap,
520 	.vm_ops = &ivpu_vm_ops,
521 	.get_sg_table = ivpu_bo_get_sg_table,
522 };
523 
524 int
ivpu_bo_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file)525 ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
526 {
527 	struct ivpu_file_priv *file_priv = file->driver_priv;
528 	struct ivpu_device *vdev = file_priv->vdev;
529 	struct drm_ivpu_bo_create *args = data;
530 	u64 size = PAGE_ALIGN(args->size);
531 	struct ivpu_bo *bo;
532 	int ret;
533 
534 	if (args->flags & ~DRM_IVPU_BO_FLAGS)
535 		return -EINVAL;
536 
537 	if (size == 0)
538 		return -EINVAL;
539 
540 	bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0);
541 	if (IS_ERR(bo)) {
542 		ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
543 			 bo, file_priv->ctx.id, args->size, args->flags);
544 		return PTR_ERR(bo);
545 	}
546 
547 	ret = drm_gem_handle_create(file, &bo->base, &bo->handle);
548 	if (!ret) {
549 		args->vpu_addr = bo->vpu_addr;
550 		args->handle = bo->handle;
551 	}
552 
553 	drm_gem_object_put(&bo->base);
554 
555 	ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n",
556 		 file_priv->ctx.id, bo->vpu_addr, bo->base.size, bo->flags);
557 
558 	return ret;
559 }
560 
561 struct ivpu_bo *
ivpu_bo_alloc_internal(struct ivpu_device * vdev,u64 vpu_addr,u64 size,u32 flags)562 ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags)
563 {
564 	const struct ivpu_addr_range *range;
565 	struct ivpu_addr_range fixed_range;
566 	struct ivpu_bo *bo;
567 	pgprot_t prot;
568 	int ret;
569 
570 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
571 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
572 
573 	if (vpu_addr) {
574 		fixed_range.start = vpu_addr;
575 		fixed_range.end = vpu_addr + size;
576 		range = &fixed_range;
577 	} else {
578 		range = &vdev->hw->ranges.global;
579 	}
580 
581 	bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0);
582 	if (IS_ERR(bo)) {
583 		ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
584 			 bo, vpu_addr, size, flags);
585 		return NULL;
586 	}
587 
588 	ret = ivpu_bo_pin(bo);
589 	if (ret)
590 		goto err_put;
591 
592 	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
593 		drm_clflush_pages(bo->pages, bo->base.size >> PAGE_SHIFT);
594 
595 	if (bo->flags & DRM_IVPU_BO_WC)
596 		set_pages_array_wc(bo->pages, bo->base.size >> PAGE_SHIFT);
597 	else if (bo->flags & DRM_IVPU_BO_UNCACHED)
598 		set_pages_array_uc(bo->pages, bo->base.size >> PAGE_SHIFT);
599 
600 	prot = ivpu_bo_pgprot(bo, PAGE_KERNEL);
601 	bo->kvaddr = vmap(bo->pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot);
602 	if (!bo->kvaddr) {
603 		ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n");
604 		goto err_put;
605 	}
606 
607 	ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n",
608 		 bo->vpu_addr, bo->base.size, flags);
609 
610 	return bo;
611 
612 err_put:
613 	drm_gem_object_put(&bo->base);
614 	return NULL;
615 }
616 
ivpu_bo_free_internal(struct ivpu_bo * bo)617 void ivpu_bo_free_internal(struct ivpu_bo *bo)
618 {
619 	drm_gem_object_put(&bo->base);
620 }
621 
ivpu_gem_prime_import(struct drm_device * dev,struct dma_buf * buf)622 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
623 {
624 	struct ivpu_device *vdev = to_ivpu_device(dev);
625 	struct dma_buf_attachment *attach;
626 	struct ivpu_bo *bo;
627 
628 	attach = dma_buf_attach(buf, dev->dev);
629 	if (IS_ERR(attach))
630 		return ERR_CAST(attach);
631 
632 	get_dma_buf(buf);
633 
634 	bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0);
635 	if (IS_ERR(bo)) {
636 		ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size);
637 		goto err_detach;
638 	}
639 
640 	lockdep_set_class(&bo->lock, &prime_bo_lock_class_key);
641 
642 	bo->base.import_attach = attach;
643 
644 	return &bo->base;
645 
646 err_detach:
647 	dma_buf_detach(buf, attach);
648 	dma_buf_put(buf);
649 	return ERR_CAST(bo);
650 }
651 
ivpu_bo_info_ioctl(struct drm_device * dev,void * data,struct drm_file * file)652 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
653 {
654 	struct ivpu_file_priv *file_priv = file->driver_priv;
655 	struct ivpu_device *vdev = to_ivpu_device(dev);
656 	struct drm_ivpu_bo_info *args = data;
657 	struct drm_gem_object *obj;
658 	struct ivpu_bo *bo;
659 	int ret = 0;
660 
661 	obj = drm_gem_object_lookup(file, args->handle);
662 	if (!obj)
663 		return -ENOENT;
664 
665 	bo = to_ivpu_bo(obj);
666 
667 	mutex_lock(&bo->lock);
668 
669 	if (!bo->ctx) {
670 		ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL);
671 		if (ret) {
672 			ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret);
673 			goto unlock;
674 		}
675 	}
676 
677 	args->flags = bo->flags;
678 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
679 	args->vpu_addr = bo->vpu_addr;
680 	args->size = obj->size;
681 unlock:
682 	mutex_unlock(&bo->lock);
683 	drm_gem_object_put(obj);
684 	return ret;
685 }
686 
ivpu_bo_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * file)687 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
688 {
689 	struct drm_ivpu_bo_wait *args = data;
690 	struct drm_gem_object *obj;
691 	unsigned long timeout;
692 	long ret;
693 
694 	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
695 
696 	obj = drm_gem_object_lookup(file, args->handle);
697 	if (!obj)
698 		return -EINVAL;
699 
700 	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
701 	if (ret == 0) {
702 		ret = -ETIMEDOUT;
703 	} else if (ret > 0) {
704 		ret = 0;
705 		args->job_status = to_ivpu_bo(obj)->job_status;
706 	}
707 
708 	drm_gem_object_put(obj);
709 
710 	return ret;
711 }
712 
ivpu_bo_print_info(struct ivpu_bo * bo,struct drm_printer * p)713 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
714 {
715 	unsigned long dma_refcount = 0;
716 
717 	if (bo->base.dma_buf && bo->base.dma_buf->file)
718 		dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count);
719 
720 	drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n",
721 		   bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size,
722 		   kref_read(&bo->base.refcount), dma_refcount, bo->ops->name);
723 }
724 
ivpu_bo_list(struct drm_device * dev,struct drm_printer * p)725 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
726 {
727 	struct ivpu_device *vdev = to_ivpu_device(dev);
728 	struct ivpu_file_priv *file_priv;
729 	unsigned long ctx_id;
730 	struct ivpu_bo *bo;
731 
732 	drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n",
733 		   "ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type");
734 
735 	mutex_lock(&vdev->gctx.lock);
736 	list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node)
737 		ivpu_bo_print_info(bo, p);
738 	mutex_unlock(&vdev->gctx.lock);
739 
740 	xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
741 		file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
742 		if (!file_priv)
743 			continue;
744 
745 		mutex_lock(&file_priv->ctx.lock);
746 		list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node)
747 			ivpu_bo_print_info(bo, p);
748 		mutex_unlock(&file_priv->ctx.lock);
749 
750 		ivpu_file_priv_put(&file_priv);
751 	}
752 }
753 
ivpu_bo_list_print(struct drm_device * dev)754 void ivpu_bo_list_print(struct drm_device *dev)
755 {
756 	struct drm_printer p = drm_info_printer(dev->dev);
757 
758 	ivpu_bo_list(dev, &p);
759 }
760