1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30  *    Dave Airlie
31  */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
35 
36 #include <drm/amdgpu_drm.h>
37 #include <drm/drm_cache.h>
38 #include "amdgpu.h"
39 #include "amdgpu_trace.h"
40 #include "amdgpu_amdkfd.h"
41 
42 /**
43  * DOC: amdgpu_object
44  *
45  * This defines the interfaces to operate on an &amdgpu_bo buffer object which
46  * represents memory used by driver (VRAM, system memory, etc.). The driver
47  * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
48  * to create/destroy/set buffer object which are then managed by the kernel TTM
49  * memory manager.
50  * The interfaces are also used internally by kernel clients, including gfx,
51  * uvd, etc. for kernel managed allocations used by the GPU.
52  *
53  */
54 
55 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
56 {
57 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
58 
59 	amdgpu_bo_kunmap(bo);
60 
61 	if (bo->tbo.base.import_attach)
62 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
63 	drm_gem_object_release(&bo->tbo.base);
64 	amdgpu_bo_unref(&bo->parent);
65 	kvfree(bo);
66 }
67 
68 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
69 {
70 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
71 	struct amdgpu_bo_user *ubo;
72 
73 	ubo = to_amdgpu_bo_user(bo);
74 	kfree(ubo->metadata);
75 	amdgpu_bo_destroy(tbo);
76 }
77 
78 static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo)
79 {
80 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
81 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
82 	struct amdgpu_bo_vm *vmbo;
83 
84 	vmbo = to_amdgpu_bo_vm(bo);
85 	/* in case amdgpu_device_recover_vram got NULL of bo->parent */
86 	if (!list_empty(&vmbo->shadow_list)) {
87 		mutex_lock(&adev->shadow_list_lock);
88 		list_del_init(&vmbo->shadow_list);
89 		mutex_unlock(&adev->shadow_list_lock);
90 	}
91 
92 	amdgpu_bo_destroy(tbo);
93 }
94 
95 /**
96  * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
97  * @bo: buffer object to be checked
98  *
99  * Uses destroy function associated with the object to determine if this is
100  * an &amdgpu_bo.
101  *
102  * Returns:
103  * true if the object belongs to &amdgpu_bo, false if not.
104  */
105 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
106 {
107 	if (bo->destroy == &amdgpu_bo_destroy ||
108 	    bo->destroy == &amdgpu_bo_user_destroy ||
109 	    bo->destroy == &amdgpu_bo_vm_destroy)
110 		return true;
111 
112 	return false;
113 }
114 
115 /**
116  * amdgpu_bo_placement_from_domain - set buffer's placement
117  * @abo: &amdgpu_bo buffer object whose placement is to be set
118  * @domain: requested domain
119  *
120  * Sets buffer's placement according to requested domain and the buffer's
121  * flags.
122  */
123 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
124 {
125 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
126 	struct ttm_placement *placement = &abo->placement;
127 	struct ttm_place *places = abo->placements;
128 	u64 flags = abo->flags;
129 	u32 c = 0;
130 
131 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
132 		unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
133 
134 		places[c].fpfn = 0;
135 		places[c].lpfn = 0;
136 		places[c].mem_type = TTM_PL_VRAM;
137 		places[c].flags = 0;
138 
139 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
140 			places[c].lpfn = visible_pfn;
141 		else
142 			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
143 
144 		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
145 			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
146 		c++;
147 	}
148 
149 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
150 		places[c].fpfn = 0;
151 		places[c].lpfn = 0;
152 		places[c].mem_type =
153 			abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
154 			AMDGPU_PL_PREEMPT : TTM_PL_TT;
155 		places[c].flags = 0;
156 		c++;
157 	}
158 
159 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
160 		places[c].fpfn = 0;
161 		places[c].lpfn = 0;
162 		places[c].mem_type = TTM_PL_SYSTEM;
163 		places[c].flags = 0;
164 		c++;
165 	}
166 
167 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
168 		places[c].fpfn = 0;
169 		places[c].lpfn = 0;
170 		places[c].mem_type = AMDGPU_PL_GDS;
171 		places[c].flags = 0;
172 		c++;
173 	}
174 
175 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
176 		places[c].fpfn = 0;
177 		places[c].lpfn = 0;
178 		places[c].mem_type = AMDGPU_PL_GWS;
179 		places[c].flags = 0;
180 		c++;
181 	}
182 
183 	if (domain & AMDGPU_GEM_DOMAIN_OA) {
184 		places[c].fpfn = 0;
185 		places[c].lpfn = 0;
186 		places[c].mem_type = AMDGPU_PL_OA;
187 		places[c].flags = 0;
188 		c++;
189 	}
190 
191 	if (!c) {
192 		places[c].fpfn = 0;
193 		places[c].lpfn = 0;
194 		places[c].mem_type = TTM_PL_SYSTEM;
195 		places[c].flags = 0;
196 		c++;
197 	}
198 
199 	BUG_ON(c >= AMDGPU_BO_MAX_PLACEMENTS);
200 
201 	placement->num_placement = c;
202 	placement->placement = places;
203 
204 	placement->num_busy_placement = c;
205 	placement->busy_placement = places;
206 }
207 
208 /**
209  * amdgpu_bo_create_reserved - create reserved BO for kernel use
210  *
211  * @adev: amdgpu device object
212  * @size: size for the new BO
213  * @align: alignment for the new BO
214  * @domain: where to place it
215  * @bo_ptr: used to initialize BOs in structures
216  * @gpu_addr: GPU addr of the pinned BO
217  * @cpu_addr: optional CPU address mapping
218  *
219  * Allocates and pins a BO for kernel internal use, and returns it still
220  * reserved.
221  *
222  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
223  *
224  * Returns:
225  * 0 on success, negative error code otherwise.
226  */
227 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
228 			      unsigned long size, int align,
229 			      u32 domain, struct amdgpu_bo **bo_ptr,
230 			      u64 *gpu_addr, void **cpu_addr)
231 {
232 	struct amdgpu_bo_param bp;
233 	bool free = false;
234 	int r;
235 
236 	if (!size) {
237 		amdgpu_bo_unref(bo_ptr);
238 		return 0;
239 	}
240 
241 	memset(&bp, 0, sizeof(bp));
242 	bp.size = size;
243 	bp.byte_align = align;
244 	bp.domain = domain;
245 	bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
246 		: AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
247 	bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
248 	bp.type = ttm_bo_type_kernel;
249 	bp.resv = NULL;
250 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
251 
252 	if (!*bo_ptr) {
253 		r = amdgpu_bo_create(adev, &bp, bo_ptr);
254 		if (r) {
255 			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
256 				r);
257 			return r;
258 		}
259 		free = true;
260 	}
261 
262 	r = amdgpu_bo_reserve(*bo_ptr, false);
263 	if (r) {
264 		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
265 		goto error_free;
266 	}
267 
268 	r = amdgpu_bo_pin(*bo_ptr, domain);
269 	if (r) {
270 		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
271 		goto error_unreserve;
272 	}
273 
274 	r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
275 	if (r) {
276 		dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
277 		goto error_unpin;
278 	}
279 
280 	if (gpu_addr)
281 		*gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
282 
283 	if (cpu_addr) {
284 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
285 		if (r) {
286 			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
287 			goto error_unpin;
288 		}
289 	}
290 
291 	return 0;
292 
293 error_unpin:
294 	amdgpu_bo_unpin(*bo_ptr);
295 error_unreserve:
296 	amdgpu_bo_unreserve(*bo_ptr);
297 
298 error_free:
299 	if (free)
300 		amdgpu_bo_unref(bo_ptr);
301 
302 	return r;
303 }
304 
305 /**
306  * amdgpu_bo_create_kernel - create BO for kernel use
307  *
308  * @adev: amdgpu device object
309  * @size: size for the new BO
310  * @align: alignment for the new BO
311  * @domain: where to place it
312  * @bo_ptr:  used to initialize BOs in structures
313  * @gpu_addr: GPU addr of the pinned BO
314  * @cpu_addr: optional CPU address mapping
315  *
316  * Allocates and pins a BO for kernel internal use.
317  *
318  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
319  *
320  * Returns:
321  * 0 on success, negative error code otherwise.
322  */
323 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
324 			    unsigned long size, int align,
325 			    u32 domain, struct amdgpu_bo **bo_ptr,
326 			    u64 *gpu_addr, void **cpu_addr)
327 {
328 	int r;
329 
330 	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
331 				      gpu_addr, cpu_addr);
332 
333 	if (r)
334 		return r;
335 
336 	if (*bo_ptr)
337 		amdgpu_bo_unreserve(*bo_ptr);
338 
339 	return 0;
340 }
341 
342 /**
343  * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
344  *
345  * @adev: amdgpu device object
346  * @offset: offset of the BO
347  * @size: size of the BO
348  * @domain: where to place it
349  * @bo_ptr:  used to initialize BOs in structures
350  * @cpu_addr: optional CPU address mapping
351  *
352  * Creates a kernel BO at a specific offset in the address space of the domain.
353  *
354  * Returns:
355  * 0 on success, negative error code otherwise.
356  */
357 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
358 			       uint64_t offset, uint64_t size, uint32_t domain,
359 			       struct amdgpu_bo **bo_ptr, void **cpu_addr)
360 {
361 	struct ttm_operation_ctx ctx = { false, false };
362 	unsigned int i;
363 	int r;
364 
365 	offset &= PAGE_MASK;
366 	size = ALIGN(size, PAGE_SIZE);
367 
368 	r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, domain, bo_ptr,
369 				      NULL, cpu_addr);
370 	if (r)
371 		return r;
372 
373 	if ((*bo_ptr) == NULL)
374 		return 0;
375 
376 	/*
377 	 * Remove the original mem node and create a new one at the request
378 	 * position.
379 	 */
380 	if (cpu_addr)
381 		amdgpu_bo_kunmap(*bo_ptr);
382 
383 	ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
384 
385 	for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
386 		(*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
387 		(*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
388 	}
389 	r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
390 			     &(*bo_ptr)->tbo.resource, &ctx);
391 	if (r)
392 		goto error;
393 
394 	if (cpu_addr) {
395 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
396 		if (r)
397 			goto error;
398 	}
399 
400 	amdgpu_bo_unreserve(*bo_ptr);
401 	return 0;
402 
403 error:
404 	amdgpu_bo_unreserve(*bo_ptr);
405 	amdgpu_bo_unref(bo_ptr);
406 	return r;
407 }
408 
409 /**
410  * amdgpu_bo_free_kernel - free BO for kernel use
411  *
412  * @bo: amdgpu BO to free
413  * @gpu_addr: pointer to where the BO's GPU memory space address was stored
414  * @cpu_addr: pointer to where the BO's CPU memory space address was stored
415  *
416  * unmaps and unpin a BO for kernel internal use.
417  */
418 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
419 			   void **cpu_addr)
420 {
421 	if (*bo == NULL)
422 		return;
423 
424 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
425 		if (cpu_addr)
426 			amdgpu_bo_kunmap(*bo);
427 
428 		amdgpu_bo_unpin(*bo);
429 		amdgpu_bo_unreserve(*bo);
430 	}
431 	amdgpu_bo_unref(bo);
432 
433 	if (gpu_addr)
434 		*gpu_addr = 0;
435 
436 	if (cpu_addr)
437 		*cpu_addr = NULL;
438 }
439 
440 /* Validate bo size is bit bigger then the request domain */
441 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
442 					  unsigned long size, u32 domain)
443 {
444 	struct ttm_resource_manager *man = NULL;
445 
446 	/*
447 	 * If GTT is part of requested domains the check must succeed to
448 	 * allow fall back to GTT
449 	 */
450 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
451 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
452 
453 		if (size < (man->size << PAGE_SHIFT))
454 			return true;
455 		else
456 			goto fail;
457 	}
458 
459 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
460 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
461 
462 		if (size < (man->size << PAGE_SHIFT))
463 			return true;
464 		else
465 			goto fail;
466 	}
467 
468 
469 	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
470 	return true;
471 
472 fail:
473 	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
474 		  man->size << PAGE_SHIFT);
475 	return false;
476 }
477 
478 bool amdgpu_bo_support_uswc(u64 bo_flags)
479 {
480 
481 #ifdef CONFIG_X86_32
482 	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
483 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
484 	 */
485 	return false;
486 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
487 	/* Don't try to enable write-combining when it can't work, or things
488 	 * may be slow
489 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
490 	 */
491 
492 #ifndef CONFIG_COMPILE_TEST
493 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
494 	 thanks to write-combining
495 #endif
496 
497 	if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
498 		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
499 			      "better performance thanks to write-combining\n");
500 	return false;
501 #else
502 	/* For architectures that don't support WC memory,
503 	 * mask out the WC flag from the BO
504 	 */
505 	if (!drm_arch_can_wc_memory())
506 		return false;
507 
508 	return true;
509 #endif
510 }
511 
512 /**
513  * amdgpu_bo_create - create an &amdgpu_bo buffer object
514  * @adev: amdgpu device object
515  * @bp: parameters to be used for the buffer object
516  * @bo_ptr: pointer to the buffer object pointer
517  *
518  * Creates an &amdgpu_bo buffer object.
519  *
520  * Returns:
521  * 0 for success or a negative error code on failure.
522  */
523 int amdgpu_bo_create(struct amdgpu_device *adev,
524 			       struct amdgpu_bo_param *bp,
525 			       struct amdgpu_bo **bo_ptr)
526 {
527 	struct ttm_operation_ctx ctx = {
528 		.interruptible = (bp->type != ttm_bo_type_kernel),
529 		.no_wait_gpu = bp->no_wait_gpu,
530 		/* We opt to avoid OOM on system pages allocations */
531 		.gfp_retry_mayfail = true,
532 		.allow_res_evict = bp->type != ttm_bo_type_kernel,
533 		.resv = bp->resv
534 	};
535 	struct amdgpu_bo *bo;
536 	unsigned long page_align, size = bp->size;
537 	int r;
538 
539 	/* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
540 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
541 		/* GWS and OA don't need any alignment. */
542 		page_align = bp->byte_align;
543 		size <<= PAGE_SHIFT;
544 	} else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
545 		/* Both size and alignment must be a multiple of 4. */
546 		page_align = ALIGN(bp->byte_align, 4);
547 		size = ALIGN(size, 4) << PAGE_SHIFT;
548 	} else {
549 		/* Memory should be aligned at least to a page size. */
550 		page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
551 		size = ALIGN(size, PAGE_SIZE);
552 	}
553 
554 	if (!amdgpu_bo_validate_size(adev, size, bp->domain))
555 		return -ENOMEM;
556 
557 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
558 
559 	*bo_ptr = NULL;
560 	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
561 	if (bo == NULL)
562 		return -ENOMEM;
563 	drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
564 	bo->vm_bo = NULL;
565 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
566 		bp->domain;
567 	bo->allowed_domains = bo->preferred_domains;
568 	if (bp->type != ttm_bo_type_kernel &&
569 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
570 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
571 
572 	bo->flags = bp->flags;
573 
574 	if (!amdgpu_bo_support_uswc(bo->flags))
575 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
576 
577 	bo->tbo.bdev = &adev->mman.bdev;
578 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
579 			  AMDGPU_GEM_DOMAIN_GDS))
580 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
581 	else
582 		amdgpu_bo_placement_from_domain(bo, bp->domain);
583 	if (bp->type == ttm_bo_type_kernel)
584 		bo->tbo.priority = 1;
585 
586 	if (!bp->destroy)
587 		bp->destroy = &amdgpu_bo_destroy;
588 
589 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
590 				 &bo->placement, page_align, &ctx,  NULL,
591 				 bp->resv, bp->destroy);
592 	if (unlikely(r != 0))
593 		return r;
594 
595 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
596 	    bo->tbo.resource->mem_type == TTM_PL_VRAM &&
597 	    bo->tbo.resource->start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
598 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
599 					     ctx.bytes_moved);
600 	else
601 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
602 
603 	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
604 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {
605 		struct dma_fence *fence;
606 
607 		r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence);
608 		if (unlikely(r))
609 			goto fail_unreserve;
610 
611 		amdgpu_bo_fence(bo, fence, false);
612 		dma_fence_put(bo->tbo.moving);
613 		bo->tbo.moving = dma_fence_get(fence);
614 		dma_fence_put(fence);
615 	}
616 	if (!bp->resv)
617 		amdgpu_bo_unreserve(bo);
618 	*bo_ptr = bo;
619 
620 	trace_amdgpu_bo_create(bo);
621 
622 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
623 	if (bp->type == ttm_bo_type_device)
624 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
625 
626 	return 0;
627 
628 fail_unreserve:
629 	if (!bp->resv)
630 		dma_resv_unlock(bo->tbo.base.resv);
631 	amdgpu_bo_unref(&bo);
632 	return r;
633 }
634 
635 /**
636  * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
637  * @adev: amdgpu device object
638  * @bp: parameters to be used for the buffer object
639  * @ubo_ptr: pointer to the buffer object pointer
640  *
641  * Create a BO to be used by user application;
642  *
643  * Returns:
644  * 0 for success or a negative error code on failure.
645  */
646 
647 int amdgpu_bo_create_user(struct amdgpu_device *adev,
648 			  struct amdgpu_bo_param *bp,
649 			  struct amdgpu_bo_user **ubo_ptr)
650 {
651 	struct amdgpu_bo *bo_ptr;
652 	int r;
653 
654 	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
655 	bp->destroy = &amdgpu_bo_user_destroy;
656 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
657 	if (r)
658 		return r;
659 
660 	*ubo_ptr = to_amdgpu_bo_user(bo_ptr);
661 	return r;
662 }
663 
664 /**
665  * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
666  * @adev: amdgpu device object
667  * @bp: parameters to be used for the buffer object
668  * @vmbo_ptr: pointer to the buffer object pointer
669  *
670  * Create a BO to be for GPUVM.
671  *
672  * Returns:
673  * 0 for success or a negative error code on failure.
674  */
675 
676 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
677 			struct amdgpu_bo_param *bp,
678 			struct amdgpu_bo_vm **vmbo_ptr)
679 {
680 	struct amdgpu_bo *bo_ptr;
681 	int r;
682 
683 	/* bo_ptr_size will be determined by the caller and it depends on
684 	 * num of amdgpu_vm_pt entries.
685 	 */
686 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
687 	bp->destroy = &amdgpu_bo_vm_destroy;
688 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
689 	if (r)
690 		return r;
691 
692 	*vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
693 	INIT_LIST_HEAD(&(*vmbo_ptr)->shadow_list);
694 	return r;
695 }
696 
697 /**
698  * amdgpu_bo_validate - validate an &amdgpu_bo buffer object
699  * @bo: pointer to the buffer object
700  *
701  * Sets placement according to domain; and changes placement and caching
702  * policy of the buffer object according to the placement.
703  * This is used for validating shadow bos.  It calls ttm_bo_validate() to
704  * make sure the buffer is resident where it needs to be.
705  *
706  * Returns:
707  * 0 for success or a negative error code on failure.
708  */
709 int amdgpu_bo_validate(struct amdgpu_bo *bo)
710 {
711 	struct ttm_operation_ctx ctx = { false, false };
712 	uint32_t domain;
713 	int r;
714 
715 	if (bo->tbo.pin_count)
716 		return 0;
717 
718 	domain = bo->preferred_domains;
719 
720 retry:
721 	amdgpu_bo_placement_from_domain(bo, domain);
722 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
723 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
724 		domain = bo->allowed_domains;
725 		goto retry;
726 	}
727 
728 	return r;
729 }
730 
731 /**
732  * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list
733  *
734  * @bo: BO that will be inserted into the shadow list
735  *
736  * Insert a BO to the shadow list.
737  */
738 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo)
739 {
740 	struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev);
741 
742 	mutex_lock(&adev->shadow_list_lock);
743 	list_add_tail(&vmbo->shadow_list, &adev->shadow_list);
744 	mutex_unlock(&adev->shadow_list_lock);
745 }
746 
747 /**
748  * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow
749  *
750  * @shadow: &amdgpu_bo shadow to be restored
751  * @fence: dma_fence associated with the operation
752  *
753  * Copies a buffer object's shadow content back to the object.
754  * This is used for recovering a buffer from its shadow in case of a gpu
755  * reset where vram context may be lost.
756  *
757  * Returns:
758  * 0 for success or a negative error code on failure.
759  */
760 int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence)
761 
762 {
763 	struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev);
764 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
765 	uint64_t shadow_addr, parent_addr;
766 
767 	shadow_addr = amdgpu_bo_gpu_offset(shadow);
768 	parent_addr = amdgpu_bo_gpu_offset(shadow->parent);
769 
770 	return amdgpu_copy_buffer(ring, shadow_addr, parent_addr,
771 				  amdgpu_bo_size(shadow), NULL, fence,
772 				  true, false, false);
773 }
774 
775 /**
776  * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
777  * @bo: &amdgpu_bo buffer object to be mapped
778  * @ptr: kernel virtual address to be returned
779  *
780  * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
781  * amdgpu_bo_kptr() to get the kernel virtual address.
782  *
783  * Returns:
784  * 0 for success or a negative error code on failure.
785  */
786 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
787 {
788 	void *kptr;
789 	long r;
790 
791 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
792 		return -EPERM;
793 
794 	kptr = amdgpu_bo_kptr(bo);
795 	if (kptr) {
796 		if (ptr)
797 			*ptr = kptr;
798 		return 0;
799 	}
800 
801 	r = dma_resv_wait_timeout(bo->tbo.base.resv, false, false,
802 				  MAX_SCHEDULE_TIMEOUT);
803 	if (r < 0)
804 		return r;
805 
806 	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.resource->num_pages, &bo->kmap);
807 	if (r)
808 		return r;
809 
810 	if (ptr)
811 		*ptr = amdgpu_bo_kptr(bo);
812 
813 	return 0;
814 }
815 
816 /**
817  * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
818  * @bo: &amdgpu_bo buffer object
819  *
820  * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
821  *
822  * Returns:
823  * the virtual address of a buffer object area.
824  */
825 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
826 {
827 	bool is_iomem;
828 
829 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
830 }
831 
832 /**
833  * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
834  * @bo: &amdgpu_bo buffer object to be unmapped
835  *
836  * Unmaps a kernel map set up by amdgpu_bo_kmap().
837  */
838 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
839 {
840 	if (bo->kmap.bo)
841 		ttm_bo_kunmap(&bo->kmap);
842 }
843 
844 /**
845  * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
846  * @bo: &amdgpu_bo buffer object
847  *
848  * References the contained &ttm_buffer_object.
849  *
850  * Returns:
851  * a refcounted pointer to the &amdgpu_bo buffer object.
852  */
853 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
854 {
855 	if (bo == NULL)
856 		return NULL;
857 
858 	ttm_bo_get(&bo->tbo);
859 	return bo;
860 }
861 
862 /**
863  * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
864  * @bo: &amdgpu_bo buffer object
865  *
866  * Unreferences the contained &ttm_buffer_object and clear the pointer
867  */
868 void amdgpu_bo_unref(struct amdgpu_bo **bo)
869 {
870 	struct ttm_buffer_object *tbo;
871 
872 	if ((*bo) == NULL)
873 		return;
874 
875 	tbo = &((*bo)->tbo);
876 	ttm_bo_put(tbo);
877 	*bo = NULL;
878 }
879 
880 /**
881  * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object
882  * @bo: &amdgpu_bo buffer object to be pinned
883  * @domain: domain to be pinned to
884  * @min_offset: the start of requested address range
885  * @max_offset: the end of requested address range
886  *
887  * Pins the buffer object according to requested domain and address range. If
888  * the memory is unbound gart memory, binds the pages into gart table. Adjusts
889  * pin_count and pin_size accordingly.
890  *
891  * Pinning means to lock pages in memory along with keeping them at a fixed
892  * offset. It is required when a buffer can not be moved, for example, when
893  * a display buffer is being scanned out.
894  *
895  * Compared with amdgpu_bo_pin(), this function gives more flexibility on
896  * where to pin a buffer if there are specific restrictions on where a buffer
897  * must be located.
898  *
899  * Returns:
900  * 0 for success or a negative error code on failure.
901  */
902 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
903 			     u64 min_offset, u64 max_offset)
904 {
905 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
906 	struct ttm_operation_ctx ctx = { false, false };
907 	int r, i;
908 
909 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
910 		return -EPERM;
911 
912 	if (WARN_ON_ONCE(min_offset > max_offset))
913 		return -EINVAL;
914 
915 	/* A shared bo cannot be migrated to VRAM */
916 	if (bo->prime_shared_count || bo->tbo.base.import_attach) {
917 		if (domain & AMDGPU_GEM_DOMAIN_GTT)
918 			domain = AMDGPU_GEM_DOMAIN_GTT;
919 		else
920 			return -EINVAL;
921 	}
922 
923 	/* This assumes only APU display buffers are pinned with (VRAM|GTT).
924 	 * See function amdgpu_display_supported_domains()
925 	 */
926 	domain = amdgpu_bo_get_preferred_pin_domain(adev, domain);
927 
928 	if (bo->tbo.pin_count) {
929 		uint32_t mem_type = bo->tbo.resource->mem_type;
930 		uint32_t mem_flags = bo->tbo.resource->placement;
931 
932 		if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
933 			return -EINVAL;
934 
935 		if ((mem_type == TTM_PL_VRAM) &&
936 		    (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
937 		    !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
938 			return -EINVAL;
939 
940 		ttm_bo_pin(&bo->tbo);
941 
942 		if (max_offset != 0) {
943 			u64 domain_start = amdgpu_ttm_domain_start(adev,
944 								   mem_type);
945 			WARN_ON_ONCE(max_offset <
946 				     (amdgpu_bo_gpu_offset(bo) - domain_start));
947 		}
948 
949 		return 0;
950 	}
951 
952 	if (bo->tbo.base.import_attach)
953 		dma_buf_pin(bo->tbo.base.import_attach);
954 
955 	/* force to pin into visible video ram */
956 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
957 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
958 	amdgpu_bo_placement_from_domain(bo, domain);
959 	for (i = 0; i < bo->placement.num_placement; i++) {
960 		unsigned fpfn, lpfn;
961 
962 		fpfn = min_offset >> PAGE_SHIFT;
963 		lpfn = max_offset >> PAGE_SHIFT;
964 
965 		if (fpfn > bo->placements[i].fpfn)
966 			bo->placements[i].fpfn = fpfn;
967 		if (!bo->placements[i].lpfn ||
968 		    (lpfn && lpfn < bo->placements[i].lpfn))
969 			bo->placements[i].lpfn = lpfn;
970 	}
971 
972 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
973 	if (unlikely(r)) {
974 		dev_err(adev->dev, "%p pin failed\n", bo);
975 		goto error;
976 	}
977 
978 	ttm_bo_pin(&bo->tbo);
979 
980 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
981 	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
982 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
983 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
984 			     &adev->visible_pin_size);
985 	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
986 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
987 	}
988 
989 error:
990 	return r;
991 }
992 
993 /**
994  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
995  * @bo: &amdgpu_bo buffer object to be pinned
996  * @domain: domain to be pinned to
997  *
998  * A simple wrapper to amdgpu_bo_pin_restricted().
999  * Provides a simpler API for buffers that do not have any strict restrictions
1000  * on where a buffer must be located.
1001  *
1002  * Returns:
1003  * 0 for success or a negative error code on failure.
1004  */
1005 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
1006 {
1007 	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1008 	return amdgpu_bo_pin_restricted(bo, domain, 0, 0);
1009 }
1010 
1011 /**
1012  * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1013  * @bo: &amdgpu_bo buffer object to be unpinned
1014  *
1015  * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1016  * Changes placement and pin size accordingly.
1017  *
1018  * Returns:
1019  * 0 for success or a negative error code on failure.
1020  */
1021 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1022 {
1023 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1024 
1025 	ttm_bo_unpin(&bo->tbo);
1026 	if (bo->tbo.pin_count)
1027 		return;
1028 
1029 	if (bo->tbo.base.import_attach)
1030 		dma_buf_unpin(bo->tbo.base.import_attach);
1031 
1032 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1033 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1034 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1035 			     &adev->visible_pin_size);
1036 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1037 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1038 	}
1039 }
1040 
1041 /**
1042  * amdgpu_bo_evict_vram - evict VRAM buffers
1043  * @adev: amdgpu device object
1044  *
1045  * Evicts all VRAM buffers on the lru list of the memory type.
1046  * Mainly used for evicting vram at suspend time.
1047  *
1048  * Returns:
1049  * 0 for success or a negative error code on failure.
1050  */
1051 int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
1052 {
1053 	struct ttm_resource_manager *man;
1054 
1055 	if (adev->in_s3 && (adev->flags & AMD_IS_APU)) {
1056 		/* No need to evict vram on APUs for suspend to ram */
1057 		return 0;
1058 	}
1059 
1060 	man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
1061 	return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
1062 }
1063 
1064 static const char *amdgpu_vram_names[] = {
1065 	"UNKNOWN",
1066 	"GDDR1",
1067 	"DDR2",
1068 	"GDDR3",
1069 	"GDDR4",
1070 	"GDDR5",
1071 	"HBM",
1072 	"DDR3",
1073 	"DDR4",
1074 	"GDDR6",
1075 	"DDR5"
1076 };
1077 
1078 /**
1079  * amdgpu_bo_init - initialize memory manager
1080  * @adev: amdgpu device object
1081  *
1082  * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1083  *
1084  * Returns:
1085  * 0 for success or a negative error code on failure.
1086  */
1087 int amdgpu_bo_init(struct amdgpu_device *adev)
1088 {
1089 	/* On A+A platform, VRAM can be mapped as WB */
1090 	if (!adev->gmc.xgmi.connected_to_cpu) {
1091 		/* reserve PAT memory space to WC for VRAM */
1092 		arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1093 				adev->gmc.aper_size);
1094 
1095 		/* Add an MTRR for the VRAM */
1096 		adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1097 				adev->gmc.aper_size);
1098 	}
1099 
1100 	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1101 		 adev->gmc.mc_vram_size >> 20,
1102 		 (unsigned long long)adev->gmc.aper_size >> 20);
1103 	DRM_INFO("RAM width %dbits %s\n",
1104 		 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1105 	return amdgpu_ttm_init(adev);
1106 }
1107 
1108 /**
1109  * amdgpu_bo_fini - tear down memory manager
1110  * @adev: amdgpu device object
1111  *
1112  * Reverses amdgpu_bo_init() to tear down memory manager.
1113  */
1114 void amdgpu_bo_fini(struct amdgpu_device *adev)
1115 {
1116 	amdgpu_ttm_fini(adev);
1117 }
1118 
1119 /**
1120  * amdgpu_bo_set_tiling_flags - set tiling flags
1121  * @bo: &amdgpu_bo buffer object
1122  * @tiling_flags: new flags
1123  *
1124  * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1125  * kernel driver to set the tiling flags on a buffer.
1126  *
1127  * Returns:
1128  * 0 for success or a negative error code on failure.
1129  */
1130 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1131 {
1132 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1133 	struct amdgpu_bo_user *ubo;
1134 
1135 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1136 	if (adev->family <= AMDGPU_FAMILY_CZ &&
1137 	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1138 		return -EINVAL;
1139 
1140 	ubo = to_amdgpu_bo_user(bo);
1141 	ubo->tiling_flags = tiling_flags;
1142 	return 0;
1143 }
1144 
1145 /**
1146  * amdgpu_bo_get_tiling_flags - get tiling flags
1147  * @bo: &amdgpu_bo buffer object
1148  * @tiling_flags: returned flags
1149  *
1150  * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1151  * set the tiling flags on a buffer.
1152  */
1153 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1154 {
1155 	struct amdgpu_bo_user *ubo;
1156 
1157 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1158 	dma_resv_assert_held(bo->tbo.base.resv);
1159 	ubo = to_amdgpu_bo_user(bo);
1160 
1161 	if (tiling_flags)
1162 		*tiling_flags = ubo->tiling_flags;
1163 }
1164 
1165 /**
1166  * amdgpu_bo_set_metadata - set metadata
1167  * @bo: &amdgpu_bo buffer object
1168  * @metadata: new metadata
1169  * @metadata_size: size of the new metadata
1170  * @flags: flags of the new metadata
1171  *
1172  * Sets buffer object's metadata, its size and flags.
1173  * Used via GEM ioctl.
1174  *
1175  * Returns:
1176  * 0 for success or a negative error code on failure.
1177  */
1178 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
1179 			    uint32_t metadata_size, uint64_t flags)
1180 {
1181 	struct amdgpu_bo_user *ubo;
1182 	void *buffer;
1183 
1184 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1185 	ubo = to_amdgpu_bo_user(bo);
1186 	if (!metadata_size) {
1187 		if (ubo->metadata_size) {
1188 			kfree(ubo->metadata);
1189 			ubo->metadata = NULL;
1190 			ubo->metadata_size = 0;
1191 		}
1192 		return 0;
1193 	}
1194 
1195 	if (metadata == NULL)
1196 		return -EINVAL;
1197 
1198 	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1199 	if (buffer == NULL)
1200 		return -ENOMEM;
1201 
1202 	kfree(ubo->metadata);
1203 	ubo->metadata_flags = flags;
1204 	ubo->metadata = buffer;
1205 	ubo->metadata_size = metadata_size;
1206 
1207 	return 0;
1208 }
1209 
1210 /**
1211  * amdgpu_bo_get_metadata - get metadata
1212  * @bo: &amdgpu_bo buffer object
1213  * @buffer: returned metadata
1214  * @buffer_size: size of the buffer
1215  * @metadata_size: size of the returned metadata
1216  * @flags: flags of the returned metadata
1217  *
1218  * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1219  * less than metadata_size.
1220  * Used via GEM ioctl.
1221  *
1222  * Returns:
1223  * 0 for success or a negative error code on failure.
1224  */
1225 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1226 			   size_t buffer_size, uint32_t *metadata_size,
1227 			   uint64_t *flags)
1228 {
1229 	struct amdgpu_bo_user *ubo;
1230 
1231 	if (!buffer && !metadata_size)
1232 		return -EINVAL;
1233 
1234 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1235 	ubo = to_amdgpu_bo_user(bo);
1236 	if (metadata_size)
1237 		*metadata_size = ubo->metadata_size;
1238 
1239 	if (buffer) {
1240 		if (buffer_size < ubo->metadata_size)
1241 			return -EINVAL;
1242 
1243 		if (ubo->metadata_size)
1244 			memcpy(buffer, ubo->metadata, ubo->metadata_size);
1245 	}
1246 
1247 	if (flags)
1248 		*flags = ubo->metadata_flags;
1249 
1250 	return 0;
1251 }
1252 
1253 /**
1254  * amdgpu_bo_move_notify - notification about a memory move
1255  * @bo: pointer to a buffer object
1256  * @evict: if this move is evicting the buffer from the graphics address space
1257  * @new_mem: new information of the bufer object
1258  *
1259  * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1260  * bookkeeping.
1261  * TTM driver callback which is called when ttm moves a buffer.
1262  */
1263 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1264 			   bool evict,
1265 			   struct ttm_resource *new_mem)
1266 {
1267 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1268 	struct amdgpu_bo *abo;
1269 	struct ttm_resource *old_mem = bo->resource;
1270 
1271 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1272 		return;
1273 
1274 	abo = ttm_to_amdgpu_bo(bo);
1275 	amdgpu_vm_bo_invalidate(adev, abo, evict);
1276 
1277 	amdgpu_bo_kunmap(abo);
1278 
1279 	if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
1280 	    bo->resource->mem_type != TTM_PL_SYSTEM)
1281 		dma_buf_move_notify(abo->tbo.base.dma_buf);
1282 
1283 	/* remember the eviction */
1284 	if (evict)
1285 		atomic64_inc(&adev->num_evictions);
1286 
1287 	/* update statistics */
1288 	if (!new_mem)
1289 		return;
1290 
1291 	/* move_notify is called before move happens */
1292 	trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
1293 }
1294 
1295 void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
1296 				uint64_t *gtt_mem, uint64_t *cpu_mem)
1297 {
1298 	unsigned int domain;
1299 
1300 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1301 	switch (domain) {
1302 	case AMDGPU_GEM_DOMAIN_VRAM:
1303 		*vram_mem += amdgpu_bo_size(bo);
1304 		break;
1305 	case AMDGPU_GEM_DOMAIN_GTT:
1306 		*gtt_mem += amdgpu_bo_size(bo);
1307 		break;
1308 	case AMDGPU_GEM_DOMAIN_CPU:
1309 	default:
1310 		*cpu_mem += amdgpu_bo_size(bo);
1311 		break;
1312 	}
1313 }
1314 
1315 /**
1316  * amdgpu_bo_release_notify - notification about a BO being released
1317  * @bo: pointer to a buffer object
1318  *
1319  * Wipes VRAM buffers whose contents should not be leaked before the
1320  * memory is released.
1321  */
1322 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1323 {
1324 	struct dma_fence *fence = NULL;
1325 	struct amdgpu_bo *abo;
1326 	int r;
1327 
1328 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1329 		return;
1330 
1331 	abo = ttm_to_amdgpu_bo(bo);
1332 
1333 	if (abo->kfd_bo)
1334 		amdgpu_amdkfd_unreserve_memory_limit(abo);
1335 
1336 	/* We only remove the fence if the resv has individualized. */
1337 	WARN_ON_ONCE(bo->type == ttm_bo_type_kernel
1338 			&& bo->base.resv != &bo->base._resv);
1339 	if (bo->base.resv == &bo->base._resv)
1340 		amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo);
1341 
1342 	if (bo->resource->mem_type != TTM_PL_VRAM ||
1343 	    !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE))
1344 		return;
1345 
1346 	dma_resv_lock(bo->base.resv, NULL);
1347 
1348 	r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence);
1349 	if (!WARN_ON(r)) {
1350 		amdgpu_bo_fence(abo, fence, false);
1351 		dma_fence_put(fence);
1352 	}
1353 
1354 	dma_resv_unlock(bo->base.resv);
1355 }
1356 
1357 /**
1358  * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1359  * @bo: pointer to a buffer object
1360  *
1361  * Notifies the driver we are taking a fault on this BO and have reserved it,
1362  * also performs bookkeeping.
1363  * TTM driver callback for dealing with vm faults.
1364  *
1365  * Returns:
1366  * 0 for success or a negative error code on failure.
1367  */
1368 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1369 {
1370 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1371 	struct ttm_operation_ctx ctx = { false, false };
1372 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1373 	unsigned long offset;
1374 	int r;
1375 
1376 	/* Remember that this BO was accessed by the CPU */
1377 	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1378 
1379 	if (bo->resource->mem_type != TTM_PL_VRAM)
1380 		return 0;
1381 
1382 	offset = bo->resource->start << PAGE_SHIFT;
1383 	if ((offset + bo->base.size) <= adev->gmc.visible_vram_size)
1384 		return 0;
1385 
1386 	/* Can't move a pinned BO to visible VRAM */
1387 	if (abo->tbo.pin_count > 0)
1388 		return VM_FAULT_SIGBUS;
1389 
1390 	/* hurrah the memory is not visible ! */
1391 	atomic64_inc(&adev->num_vram_cpu_page_faults);
1392 	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1393 					AMDGPU_GEM_DOMAIN_GTT);
1394 
1395 	/* Avoid costly evictions; only set GTT as a busy placement */
1396 	abo->placement.num_busy_placement = 1;
1397 	abo->placement.busy_placement = &abo->placements[1];
1398 
1399 	r = ttm_bo_validate(bo, &abo->placement, &ctx);
1400 	if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1401 		return VM_FAULT_NOPAGE;
1402 	else if (unlikely(r))
1403 		return VM_FAULT_SIGBUS;
1404 
1405 	offset = bo->resource->start << PAGE_SHIFT;
1406 	/* this should never happen */
1407 	if (bo->resource->mem_type == TTM_PL_VRAM &&
1408 	    (offset + bo->base.size) > adev->gmc.visible_vram_size)
1409 		return VM_FAULT_SIGBUS;
1410 
1411 	ttm_bo_move_to_lru_tail_unlocked(bo);
1412 	return 0;
1413 }
1414 
1415 /**
1416  * amdgpu_bo_fence - add fence to buffer object
1417  *
1418  * @bo: buffer object in question
1419  * @fence: fence to add
1420  * @shared: true if fence should be added shared
1421  *
1422  */
1423 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1424 		     bool shared)
1425 {
1426 	struct dma_resv *resv = bo->tbo.base.resv;
1427 
1428 	if (shared)
1429 		dma_resv_add_shared_fence(resv, fence);
1430 	else
1431 		dma_resv_add_excl_fence(resv, fence);
1432 }
1433 
1434 /**
1435  * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1436  *
1437  * @adev: amdgpu device pointer
1438  * @resv: reservation object to sync to
1439  * @sync_mode: synchronization mode
1440  * @owner: fence owner
1441  * @intr: Whether the wait is interruptible
1442  *
1443  * Extract the fences from the reservation object and waits for them to finish.
1444  *
1445  * Returns:
1446  * 0 on success, errno otherwise.
1447  */
1448 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1449 			     enum amdgpu_sync_mode sync_mode, void *owner,
1450 			     bool intr)
1451 {
1452 	struct amdgpu_sync sync;
1453 	int r;
1454 
1455 	amdgpu_sync_create(&sync);
1456 	amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1457 	r = amdgpu_sync_wait(&sync, intr);
1458 	amdgpu_sync_free(&sync);
1459 	return r;
1460 }
1461 
1462 /**
1463  * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1464  * @bo: buffer object to wait for
1465  * @owner: fence owner
1466  * @intr: Whether the wait is interruptible
1467  *
1468  * Wrapper to wait for fences in a BO.
1469  * Returns:
1470  * 0 on success, errno otherwise.
1471  */
1472 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1473 {
1474 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1475 
1476 	return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1477 					AMDGPU_SYNC_NE_OWNER, owner, intr);
1478 }
1479 
1480 /**
1481  * amdgpu_bo_gpu_offset - return GPU offset of bo
1482  * @bo:	amdgpu object for which we query the offset
1483  *
1484  * Note: object should either be pinned or reserved when calling this
1485  * function, it might be useful to add check for this for debugging.
1486  *
1487  * Returns:
1488  * current GPU offset of the object.
1489  */
1490 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1491 {
1492 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1493 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1494 		     !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1495 	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1496 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1497 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1498 
1499 	return amdgpu_bo_gpu_offset_no_check(bo);
1500 }
1501 
1502 /**
1503  * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1504  * @bo:	amdgpu object for which we query the offset
1505  *
1506  * Returns:
1507  * current GPU offset of the object without raising warnings.
1508  */
1509 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1510 {
1511 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1512 	uint64_t offset;
1513 
1514 	offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1515 		 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1516 
1517 	return amdgpu_gmc_sign_extend(offset);
1518 }
1519 
1520 /**
1521  * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout
1522  * @adev: amdgpu device object
1523  * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1524  *
1525  * Returns:
1526  * Which of the allowed domains is preferred for pinning the BO for scanout.
1527  */
1528 uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev,
1529 					    uint32_t domain)
1530 {
1531 	if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
1532 		domain = AMDGPU_GEM_DOMAIN_VRAM;
1533 		if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1534 			domain = AMDGPU_GEM_DOMAIN_GTT;
1535 	}
1536 	return domain;
1537 }
1538 
1539 #if defined(CONFIG_DEBUG_FS)
1540 #define amdgpu_bo_print_flag(m, bo, flag)		        \
1541 	do {							\
1542 		if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
1543 			seq_printf((m), " " #flag);		\
1544 		}						\
1545 	} while (0)
1546 
1547 /**
1548  * amdgpu_bo_print_info - print BO info in debugfs file
1549  *
1550  * @id: Index or Id of the BO
1551  * @bo: Requested BO for printing info
1552  * @m: debugfs file
1553  *
1554  * Print BO information in debugfs file
1555  *
1556  * Returns:
1557  * Size of the BO in bytes.
1558  */
1559 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1560 {
1561 	struct dma_buf_attachment *attachment;
1562 	struct dma_buf *dma_buf;
1563 	unsigned int domain;
1564 	const char *placement;
1565 	unsigned int pin_count;
1566 	u64 size;
1567 
1568 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1569 	switch (domain) {
1570 	case AMDGPU_GEM_DOMAIN_VRAM:
1571 		placement = "VRAM";
1572 		break;
1573 	case AMDGPU_GEM_DOMAIN_GTT:
1574 		placement = " GTT";
1575 		break;
1576 	case AMDGPU_GEM_DOMAIN_CPU:
1577 	default:
1578 		placement = " CPU";
1579 		break;
1580 	}
1581 
1582 	size = amdgpu_bo_size(bo);
1583 	seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1584 			id, size, placement);
1585 
1586 	pin_count = READ_ONCE(bo->tbo.pin_count);
1587 	if (pin_count)
1588 		seq_printf(m, " pin count %d", pin_count);
1589 
1590 	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1591 	attachment = READ_ONCE(bo->tbo.base.import_attach);
1592 
1593 	if (attachment)
1594 		seq_printf(m, " imported from %p", dma_buf);
1595 	else if (dma_buf)
1596 		seq_printf(m, " exported as %p", dma_buf);
1597 
1598 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1599 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1600 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1601 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1602 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1603 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1604 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1605 
1606 	seq_puts(m, "\n");
1607 
1608 	return size;
1609 }
1610 #endif
1611